HMVMXR0.cpp@ 67035

Last change on this file since 67035 was 67026, checked in by vboxsync, 8 years ago
VMM/HMVMXR0: Fixes in event reflection with pending events.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 597.1 KB

Line
1	/* $Id: HMVMXR0.cpp 67026 2017-05-23 07:49:33Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2016 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#define HMVMX_USE_IEM_EVENT_REFLECTION
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_SWAP_FPU_STATE
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually except:
142	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144	* due to bugs in Intel CPUs.
145	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
146	* support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*********************************************************************************************************************************
197	* Structures and Typedefs *
198	*********************************************************************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG fEFlags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit code qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u7Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringDoublePF;
278	/** Whether the VM-exit was caused by a page-fault during delivery of an
279	* external interrupt or NMI. */
280	bool fVectoringPF;
281	} VMXTRANSIENT;
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
285	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
286	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
287	/** Pointer to VMX transient state. */
288	typedef VMXTRANSIENT *PVMXTRANSIENT;
289
290
291	/**
292	* MSR-bitmap read permissions.
293	*/
294	typedef enum VMXMSREXITREAD
295	{
296	/** Reading this MSR causes a VM-exit. */
297	VMXMSREXIT_INTERCEPT_READ = 0xb,
298	/** Reading this MSR does not cause a VM-exit. */
299	VMXMSREXIT_PASSTHRU_READ
300	} VMXMSREXITREAD;
301	/** Pointer to MSR-bitmap read permissions. */
302	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
303
304	/**
305	* MSR-bitmap write permissions.
306	*/
307	typedef enum VMXMSREXITWRITE
308	{
309	/** Writing to this MSR causes a VM-exit. */
310	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
311	/** Writing to this MSR does not cause a VM-exit. */
312	VMXMSREXIT_PASSTHRU_WRITE
313	} VMXMSREXITWRITE;
314	/** Pointer to MSR-bitmap write permissions. */
315	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
316
317
318	/**
319	* VMX VM-exit handler.
320	*
321	* @returns Strict VBox status code (i.e. informational status codes too).
322	* @param pVCpu The cross context virtual CPU structure.
323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
324	* out-of-sync. Make sure to update the required
325	* fields before using them.
326	* @param pVmxTransient Pointer to the VMX-transient structure.
327	*/
328	#ifndef HMVMX_USE_FUNCTION_TABLE
329	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330	#else
331	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332	/** Pointer to VM-exit handler. */
333	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
334	#endif
335
336	/**
337	* VMX VM-exit handler, non-strict status code.
338	*
339	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
340	*
341	* @returns VBox status code, no informational status code returned.
342	* @param pVCpu The cross context virtual CPU structure.
343	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
344	* out-of-sync. Make sure to update the required
345	* fields before using them.
346	* @param pVmxTransient Pointer to the VMX-transient structure.
347	*
348	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
349	* use of that status code will be replaced with VINF_EM_SOMETHING
350	* later when switching over to IEM.
351	*/
352	#ifndef HMVMX_USE_FUNCTION_TABLE
353	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
354	#else
355	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
356	#endif
357
358
359	/*********************************************************************************************************************************
360	* Internal Functions *
361	*********************************************************************************************************************************/
362	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
363	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
364	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
365	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
366	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
367	bool fStepping, uint32_t *puIntState);
368	#if HC_ARCH_BITS == 32
369	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
370	#endif
371	#ifndef HMVMX_USE_FUNCTION_TABLE
372	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
373	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
374	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
375	#else
376	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
377	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
378	#endif
379
380
381	/** @name VM-exit handlers.
382	* @{
383	*/
384	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396	static FNVMXEXITHANDLER hmR0VmxExitHlt;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
404	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
405	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
406	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
407	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
408	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
412	static FNVMXEXITHANDLER hmR0VmxExitMwait;
413	static FNVMXEXITHANDLER hmR0VmxExitMtf;
414	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
415	static FNVMXEXITHANDLER hmR0VmxExitPause;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
417	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
418	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
419	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
422	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
423	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
424	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
425	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
426	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
427	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
428	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
429	/** @} */
430
431	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
440
441
442	/*********************************************************************************************************************************
443	* Global Variables *
444	*********************************************************************************************************************************/
445	#ifdef HMVMX_USE_FUNCTION_TABLE
446
447	/**
448	* VMX_EXIT dispatch table.
449	*/
450	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
451	{
452	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
453	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
454	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
455	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
456	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
457	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
458	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
459	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
460	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
461	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
462	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
463	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
464	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
465	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
466	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
467	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
468	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
469	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
470	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
471	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
472	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
473	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
474	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
475	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
476	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
477	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
478	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
479	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
480	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
481	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
482	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
483	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
484	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
485	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
486	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
487	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
488	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
489	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
490	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
491	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
492	/* 40 UNDEFINED */ hmR0VmxExitPause,
493	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
494	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
495	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
496	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
497	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
498	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
499	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
500	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
501	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
502	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
503	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
504	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
505	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
506	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
507	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
508	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
509	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
510	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
511	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
512	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
513	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
514	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
515	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
516	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
517	};
518	#endif /* HMVMX_USE_FUNCTION_TABLE */
519
520	#ifdef VBOX_STRICT
521	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
522	{
523	/* 0 */ "(Not Used)",
524	/* 1 */ "VMCALL executed in VMX root operation.",
525	/* 2 */ "VMCLEAR with invalid physical address.",
526	/* 3 */ "VMCLEAR with VMXON pointer.",
527	/* 4 */ "VMLAUNCH with non-clear VMCS.",
528	/* 5 */ "VMRESUME with non-launched VMCS.",
529	/* 6 */ "VMRESUME after VMXOFF",
530	/* 7 */ "VM-entry with invalid control fields.",
531	/* 8 */ "VM-entry with invalid host state fields.",
532	/* 9 */ "VMPTRLD with invalid physical address.",
533	/* 10 */ "VMPTRLD with VMXON pointer.",
534	/* 11 */ "VMPTRLD with incorrect revision identifier.",
535	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
536	/* 13 */ "VMWRITE to read-only VMCS component.",
537	/* 14 */ "(Not Used)",
538	/* 15 */ "VMXON executed in VMX root operation.",
539	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
540	/* 17 */ "VM-entry with non-launched executing VMCS.",
541	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
542	/* 19 */ "VMCALL with non-clear VMCS.",
543	/* 20 */ "VMCALL with invalid VM-exit control fields.",
544	/* 21 */ "(Not Used)",
545	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
546	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
547	/* 24 */ "VMCALL with invalid SMM-monitor features.",
548	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
549	/* 26 */ "VM-entry with events blocked by MOV SS.",
550	/* 27 */ "(Not Used)",
551	/* 28 */ "Invalid operand to INVEPT/INVVPID."
552	};
553	#endif /* VBOX_STRICT */
554
555
556
557	/**
558	* Updates the VM's last error record.
559	*
560	* If there was a VMX instruction error, reads the error data from the VMCS and
561	* updates VCPU's last error record as well.
562	*
563	* @param pVM The cross context VM structure.
564	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
565	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
566	* VERR_VMX_INVALID_VMCS_FIELD.
567	* @param rc The error code.
568	*/
569	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
570	{
571	AssertPtr(pVM);
572	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
573	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
574	{
575	AssertPtrReturnVoid(pVCpu);
576	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
577	}
578	pVM->hm.s.lLastError = rc;
579	}
580
581
582	/**
583	* Reads the VM-entry interruption-information field from the VMCS into the VMX
584	* transient structure.
585	*
586	* @returns VBox status code.
587	* @param pVmxTransient Pointer to the VMX transient structure.
588	*
589	* @remarks No-long-jump zone!!!
590	*/
591	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
592	{
593	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
594	AssertRCReturn(rc, rc);
595	return VINF_SUCCESS;
596	}
597
598
599	#ifdef VBOX_STRICT
600	/**
601	* Reads the VM-entry exception error code field from the VMCS into
602	* the VMX transient structure.
603	*
604	* @returns VBox status code.
605	* @param pVmxTransient Pointer to the VMX transient structure.
606	*
607	* @remarks No-long-jump zone!!!
608	*/
609	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612	AssertRCReturn(rc, rc);
613	return VINF_SUCCESS;
614	}
615	#endif /* VBOX_STRICT */
616
617
618	#ifdef VBOX_STRICT
619	/**
620	* Reads the VM-entry exception error code field from the VMCS into
621	* the VMX transient structure.
622	*
623	* @returns VBox status code.
624	* @param pVmxTransient Pointer to the VMX transient structure.
625	*
626	* @remarks No-long-jump zone!!!
627	*/
628	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
629	{
630	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
631	AssertRCReturn(rc, rc);
632	return VINF_SUCCESS;
633	}
634	#endif /* VBOX_STRICT */
635
636
637	/**
638	* Reads the VM-exit interruption-information field from the VMCS into the VMX
639	* transient structure.
640	*
641	* @returns VBox status code.
642	* @param pVmxTransient Pointer to the VMX transient structure.
643	*/
644	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
645	{
646	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
647	{
648	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
649	AssertRCReturn(rc, rc);
650	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
651	}
652	return VINF_SUCCESS;
653	}
654
655
656	/**
657	* Reads the VM-exit interruption error code from the VMCS into the VMX
658	* transient structure.
659	*
660	* @returns VBox status code.
661	* @param pVmxTransient Pointer to the VMX transient structure.
662	*/
663	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
664	{
665	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
666	{
667	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
668	AssertRCReturn(rc, rc);
669	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
670	}
671	return VINF_SUCCESS;
672	}
673
674
675	/**
676	* Reads the VM-exit instruction length field from the VMCS into the VMX
677	* transient structure.
678	*
679	* @returns VBox status code.
680	* @param pVmxTransient Pointer to the VMX transient structure.
681	*/
682	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
683	{
684	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
685	{
686	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
687	AssertRCReturn(rc, rc);
688	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
689	}
690	return VINF_SUCCESS;
691	}
692
693
694	/**
695	* Reads the VM-exit instruction-information field from the VMCS into
696	* the VMX transient structure.
697	*
698	* @returns VBox status code.
699	* @param pVmxTransient Pointer to the VMX transient structure.
700	*/
701	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
702	{
703	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
704	{
705	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
706	AssertRCReturn(rc, rc);
707	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
708	}
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Reads the exit code qualification from the VMCS into the VMX transient
715	* structure.
716	*
717	* @returns VBox status code.
718	* @param pVCpu The cross context virtual CPU structure of the
719	* calling EMT. (Required for the VMCS cache case.)
720	* @param pVmxTransient Pointer to the VMX transient structure.
721	*/
722	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
723	{
724	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
725	{
726	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
727	AssertRCReturn(rc, rc);
728	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
729	}
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Reads the IDT-vectoring information field from the VMCS into the VMX
736	* transient structure.
737	*
738	* @returns VBox status code.
739	* @param pVmxTransient Pointer to the VMX transient structure.
740	*
741	* @remarks No-long-jump zone!!!
742	*/
743	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
744	{
745	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
746	{
747	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
748	AssertRCReturn(rc, rc);
749	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
750	}
751	return VINF_SUCCESS;
752	}
753
754
755	/**
756	* Reads the IDT-vectoring error code from the VMCS into the VMX
757	* transient structure.
758	*
759	* @returns VBox status code.
760	* @param pVmxTransient Pointer to the VMX transient structure.
761	*/
762	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
763	{
764	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
765	{
766	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
767	AssertRCReturn(rc, rc);
768	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
769	}
770	return VINF_SUCCESS;
771	}
772
773
774	/**
775	* Enters VMX root mode operation on the current CPU.
776	*
777	* @returns VBox status code.
778	* @param pVM The cross context VM structure. Can be
779	* NULL, after a resume.
780	* @param HCPhysCpuPage Physical address of the VMXON region.
781	* @param pvCpuPage Pointer to the VMXON region.
782	*/
783	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
784	{
785	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
786	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
787	Assert(pvCpuPage);
788	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
789
790	if (pVM)
791	{
792	/* Write the VMCS revision dword to the VMXON region. */
793	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
794	}
795
796	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
797	RTCCUINTREG fEFlags = ASMIntDisableFlags();
798
799	/* Enable the VMX bit in CR4 if necessary. */
800	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
801
802	/* Enter VMX root mode. */
803	int rc = VMXEnable(HCPhysCpuPage);
804	if (RT_FAILURE(rc))
805	{
806	if (!(uOldCr4 & X86_CR4_VMXE))
807	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
808
809	if (pVM)
810	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
811	}
812
813	/* Restore interrupts. */
814	ASMSetFlags(fEFlags);
815	return rc;
816	}
817
818
819	/**
820	* Exits VMX root mode operation on the current CPU.
821	*
822	* @returns VBox status code.
823	*/
824	static int hmR0VmxLeaveRootMode(void)
825	{
826	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
827
828	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
829	RTCCUINTREG fEFlags = ASMIntDisableFlags();
830
831	/* If we're for some reason not in VMX root mode, then don't leave it. */
832	RTCCUINTREG uHostCR4 = ASMGetCR4();
833
834	int rc;
835	if (uHostCR4 & X86_CR4_VMXE)
836	{
837	/* Exit VMX root mode and clear the VMX bit in CR4. */
838	VMXDisable();
839	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
840	rc = VINF_SUCCESS;
841	}
842	else
843	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
844
845	/* Restore interrupts. */
846	ASMSetFlags(fEFlags);
847	return rc;
848	}
849
850
851	/**
852	* Allocates and maps one physically contiguous page. The allocated page is
853	* zero'd out. (Used by various VT-x structures).
854	*
855	* @returns IPRT status code.
856	* @param pMemObj Pointer to the ring-0 memory object.
857	* @param ppVirt Where to store the virtual address of the
858	* allocation.
859	* @param pHCPhys Where to store the physical address of the
860	* allocation.
861	*/
862	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
863	{
864	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
865	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
866	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
867
868	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
869	if (RT_FAILURE(rc))
870	return rc;
871	ppVirt = RTR0MemObjAddress(pMemObj);
872	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
873	ASMMemZero32(*ppVirt, PAGE_SIZE);
874	return VINF_SUCCESS;
875	}
876
877
878	/**
879	* Frees and unmaps an allocated physical page.
880	*
881	* @param pMemObj Pointer to the ring-0 memory object.
882	* @param ppVirt Where to re-initialize the virtual address of
883	* allocation as 0.
884	* @param pHCPhys Where to re-initialize the physical address of the
885	* allocation as 0.
886	*/
887	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
888	{
889	AssertPtr(pMemObj);
890	AssertPtr(ppVirt);
891	AssertPtr(pHCPhys);
892	if (*pMemObj != NIL_RTR0MEMOBJ)
893	{
894	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
895	AssertRC(rc);
896	*pMemObj = NIL_RTR0MEMOBJ;
897	*ppVirt = 0;
898	*pHCPhys = 0;
899	}
900	}
901
902
903	/**
904	* Worker function to free VT-x related structures.
905	*
906	* @returns IPRT status code.
907	* @param pVM The cross context VM structure.
908	*/
909	static void hmR0VmxStructsFree(PVM pVM)
910	{
911	for (VMCPUID i = 0; i < pVM->cCpus; i++)
912	{
913	PVMCPU pVCpu = &pVM->aCpus[i];
914	AssertPtr(pVCpu);
915
916	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
917	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
918
919	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
921
922	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923	}
924
925	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928	#endif
929	}
930
931
932	/**
933	* Worker function to allocate VT-x related VM structures.
934	*
935	* @returns IPRT status code.
936	* @param pVM The cross context VM structure.
937	*/
938	static int hmR0VmxStructsAlloc(PVM pVM)
939	{
940	/*
941	* Initialize members up-front so we can cleanup properly on allocation failure.
942	*/
943	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946	pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955	#endif
956	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959	for (VMCPUID i = 0; i < pVM->cCpus; i++)
960	{
961	PVMCPU pVCpu = &pVM->aCpus[i];
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
965	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
966	}
967	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
968	#undef VMXLOCAL_INIT_VM_MEMOBJ
969
970	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
971	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
972	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
973	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
974
975	/*
976	* Allocate all the VT-x structures.
977	*/
978	int rc = VINF_SUCCESS;
979	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
980	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
981	if (RT_FAILURE(rc))
982	goto cleanup;
983	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
984	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
985	#endif
986
987	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
988	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
989	{
990	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
991	&pVM->hm.s.vmx.HCPhysApicAccess);
992	if (RT_FAILURE(rc))
993	goto cleanup;
994	}
995
996	/*
997	* Initialize per-VCPU VT-x structures.
998	*/
999	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1000	{
1001	PVMCPU pVCpu = &pVM->aCpus[i];
1002	AssertPtr(pVCpu);
1003
1004	/* Allocate the VM control structure (VMCS). */
1005	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1006	if (RT_FAILURE(rc))
1007	goto cleanup;
1008
1009	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1010	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1011	{
1012	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013	NULL /* pR3Ptr /, NULL / pRCPtr */);
1014	if (RT_FAILURE(rc))
1015	goto cleanup;
1016	}
1017
1018	/*
1019	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020	* transparent accesses of specific MSRs.
1021	*
1022	* If the condition for enabling MSR bitmaps changes here, don't forget to
1023	* update HMAreMsrBitmapsAvailable().
1024	*/
1025	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026	{
1027	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029	if (RT_FAILURE(rc))
1030	goto cleanup;
1031	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032	}
1033
1034	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036	if (RT_FAILURE(rc))
1037	goto cleanup;
1038
1039	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1040	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041	if (RT_FAILURE(rc))
1042	goto cleanup;
1043	}
1044
1045	return VINF_SUCCESS;
1046
1047	cleanup:
1048	hmR0VmxStructsFree(pVM);
1049	return rc;
1050	}
1051
1052
1053	/**
1054	* Does global VT-x initialization (called during module initialization).
1055	*
1056	* @returns VBox status code.
1057	*/
1058	VMMR0DECL(int) VMXR0GlobalInit(void)
1059	{
1060	#ifdef HMVMX_USE_FUNCTION_TABLE
1061	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062	# ifdef VBOX_STRICT
1063	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064	Assert(g_apfnVMExitHandlers[i]);
1065	# endif
1066	#endif
1067	return VINF_SUCCESS;
1068	}
1069
1070
1071	/**
1072	* Does global VT-x termination (called during module termination).
1073	*/
1074	VMMR0DECL(void) VMXR0GlobalTerm()
1075	{
1076	/* Nothing to do currently. */
1077	}
1078
1079
1080	/**
1081	* Sets up and activates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pVM The cross context VM structure. Can be
1086	* NULL after a host resume operation.
1087	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088	* fEnabledByHost is @c true).
1089	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090	* @a fEnabledByHost is @c true).
1091	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092	* enable VT-x on the host.
1093	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1094	*/
1095	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096	void *pvMsrs)
1097	{
1098	Assert(pCpu);
1099	Assert(pvMsrs);
1100	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102	/* Enable VT-x if it's not already enabled by the host. */
1103	if (!fEnabledByHost)
1104	{
1105	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106	if (RT_FAILURE(rc))
1107	return rc;
1108	}
1109
1110	/*
1111	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1112	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1113	*/
1114	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1115	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1116	{
1117	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1118	pCpu->fFlushAsidBeforeUse = false;
1119	}
1120	else
1121	pCpu->fFlushAsidBeforeUse = true;
1122
1123	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1124	++pCpu->cTlbFlushes;
1125
1126	return VINF_SUCCESS;
1127	}
1128
1129
1130	/**
1131	* Deactivates VT-x on the current CPU.
1132	*
1133	* @returns VBox status code.
1134	* @param pCpu Pointer to the global CPU info struct.
1135	* @param pvCpuPage Pointer to the VMXON region.
1136	* @param HCPhysCpuPage Physical address of the VMXON region.
1137	*
1138	* @remarks This function should never be called when SUPR0EnableVTx() or
1139	* similar was used to enable VT-x on the host.
1140	*/
1141	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1142	{
1143	NOREF(pCpu);
1144	NOREF(pvCpuPage);
1145	NOREF(HCPhysCpuPage);
1146
1147	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1148	return hmR0VmxLeaveRootMode();
1149	}
1150
1151
1152	/**
1153	* Sets the permission bits for the specified MSR in the MSR bitmap.
1154	*
1155	* @param pVCpu The cross context virtual CPU structure.
1156	* @param uMsr The MSR value.
1157	* @param enmRead Whether reading this MSR causes a VM-exit.
1158	* @param enmWrite Whether writing this MSR causes a VM-exit.
1159	*/
1160	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1161	{
1162	int32_t iBit;
1163	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1164
1165	/*
1166	* Layout:
1167	* 0x000 - 0x3ff - Low MSR read bits
1168	* 0x400 - 0x7ff - High MSR read bits
1169	* 0x800 - 0xbff - Low MSR write bits
1170	* 0xc00 - 0xfff - High MSR write bits
1171	*/
1172	if (uMsr <= 0x00001FFF)
1173	iBit = uMsr;
1174	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1175	{
1176	iBit = uMsr - UINT32_C(0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1181
1182	Assert(iBit <= 0x1fff);
1183	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1184	ASMBitSet(pbMsrBitmap, iBit);
1185	else
1186	ASMBitClear(pbMsrBitmap, iBit);
1187
1188	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1189	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1190	else
1191	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1192	}
1193
1194
1195	#ifdef VBOX_STRICT
1196	/**
1197	* Gets the permission bits for the specified MSR in the MSR bitmap.
1198	*
1199	* @returns VBox status code.
1200	* @retval VINF_SUCCESS if the specified MSR is found.
1201	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1202	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1203	*
1204	* @param pVCpu The cross context virtual CPU structure.
1205	* @param uMsr The MSR.
1206	* @param penmRead Where to store the read permissions.
1207	* @param penmWrite Where to store the write permissions.
1208	*/
1209	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1210	{
1211	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1212	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1213	int32_t iBit;
1214	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1215
1216	/* See hmR0VmxSetMsrPermission() for the layout. */
1217	if (uMsr <= 0x00001FFF)
1218	iBit = uMsr;
1219	else if ( uMsr >= 0xC0000000
1220	&& uMsr <= 0xC0001FFF)
1221	{
1222	iBit = (uMsr - 0xC0000000);
1223	pbMsrBitmap += 0x400;
1224	}
1225	else
1226	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1227
1228	Assert(iBit <= 0x1fff);
1229	if (ASMBitTest(pbMsrBitmap, iBit))
1230	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1231	else
1232	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1233
1234	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1235	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1236	else
1237	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1238	return VINF_SUCCESS;
1239	}
1240	#endif /* VBOX_STRICT */
1241
1242
1243	/**
1244	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1245	* area.
1246	*
1247	* @returns VBox status code.
1248	* @param pVCpu The cross context virtual CPU structure.
1249	* @param cMsrs The number of MSRs.
1250	*/
1251	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1252	{
1253	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1254	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1255	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1256	{
1257	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1258	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1259	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1260	}
1261
1262	/* Update number of guest MSRs to load/store across the world-switch. */
1263	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1264	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1265
1266	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1267	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1268	AssertRCReturn(rc, rc);
1269
1270	/* Update the VCPU's copy of the MSR count. */
1271	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1272
1273	return VINF_SUCCESS;
1274	}
1275
1276
1277	/**
1278	* Adds a new (or updates the value of an existing) guest/host MSR
1279	* pair to be swapped during the world-switch as part of the
1280	* auto-load/store MSR area in the VMCS.
1281	*
1282	* @returns VBox status code.
1283	* @param pVCpu The cross context virtual CPU structure.
1284	* @param uMsr The MSR.
1285	* @param uGuestMsrValue Value of the guest MSR.
1286	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1287	* necessary.
1288	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1289	* its value was updated. Optional, can be NULL.
1290	*/
1291	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1292	bool *pfAddedAndUpdated)
1293	{
1294	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1295	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1296	uint32_t i;
1297	for (i = 0; i < cMsrs; i++)
1298	{
1299	if (pGuestMsr->u32Msr == uMsr)
1300	break;
1301	pGuestMsr++;
1302	}
1303
1304	bool fAdded = false;
1305	if (i == cMsrs)
1306	{
1307	++cMsrs;
1308	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1309	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1310
1311	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1312	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1313	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1314
1315	fAdded = true;
1316	}
1317
1318	/* Update the MSR values in the auto-load/store MSR area. */
1319	pGuestMsr->u32Msr = uMsr;
1320	pGuestMsr->u64Value = uGuestMsrValue;
1321
1322	/* Create/update the MSR slot in the host MSR area. */
1323	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1324	pHostMsr += i;
1325	pHostMsr->u32Msr = uMsr;
1326
1327	/*
1328	* Update the host MSR only when requested by the caller AND when we're
1329	* adding it to the auto-load/store area. Otherwise, it would have been
1330	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1331	*/
1332	bool fUpdatedMsrValue = false;
1333	if ( fAdded
1334	&& fUpdateHostMsr)
1335	{
1336	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1337	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1338	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1339	fUpdatedMsrValue = true;
1340	}
1341
1342	if (pfAddedAndUpdated)
1343	*pfAddedAndUpdated = fUpdatedMsrValue;
1344	return VINF_SUCCESS;
1345	}
1346
1347
1348	/**
1349	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1350	* auto-load/store MSR area in the VMCS.
1351	*
1352	* @returns VBox status code.
1353	* @param pVCpu The cross context virtual CPU structure.
1354	* @param uMsr The MSR.
1355	*/
1356	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1357	{
1358	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1359	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1360	for (uint32_t i = 0; i < cMsrs; i++)
1361	{
1362	/* Find the MSR. */
1363	if (pGuestMsr->u32Msr == uMsr)
1364	{
1365	/* If it's the last MSR, simply reduce the count. */
1366	if (i == cMsrs - 1)
1367	{
1368	--cMsrs;
1369	break;
1370	}
1371
1372	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1373	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1374	pLastGuestMsr += cMsrs - 1;
1375	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1376	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1377
1378	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380	pLastHostMsr += cMsrs - 1;
1381	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1382	pHostMsr->u64Value = pLastHostMsr->u64Value;
1383	--cMsrs;
1384	break;
1385	}
1386	pGuestMsr++;
1387	}
1388
1389	/* Update the VMCS if the count changed (meaning the MSR was found). */
1390	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1391	{
1392	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1393	AssertRCReturn(rc, rc);
1394
1395	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1396	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1397	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1398
1399	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1400	return VINF_SUCCESS;
1401	}
1402
1403	return VERR_NOT_FOUND;
1404	}
1405
1406
1407	/**
1408	* Checks if the specified guest MSR is part of the auto-load/store area in
1409	* the VMCS.
1410	*
1411	* @returns true if found, false otherwise.
1412	* @param pVCpu The cross context virtual CPU structure.
1413	* @param uMsr The MSR to find.
1414	*/
1415	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1416	{
1417	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1418	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1419
1420	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1421	{
1422	if (pGuestMsr->u32Msr == uMsr)
1423	return true;
1424	}
1425	return false;
1426	}
1427
1428
1429	/**
1430	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1431	*
1432	* @param pVCpu The cross context virtual CPU structure.
1433	*
1434	* @remarks No-long-jump zone!!!
1435	*/
1436	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1437	{
1438	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1439	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1440	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1441	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1442
1443	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1444	{
1445	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1446
1447	/*
1448	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1449	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1450	*/
1451	if (pHostMsr->u32Msr == MSR_K6_EFER)
1452	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1453	else
1454	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1455	}
1456
1457	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1458	}
1459
1460
1461	/**
1462	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1463	* perform lazy restoration of the host MSRs while leaving VT-x.
1464	*
1465	* @param pVCpu The cross context virtual CPU structure.
1466	*
1467	* @remarks No-long-jump zone!!!
1468	*/
1469	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1470	{
1471	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1472
1473	/*
1474	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1475	*/
1476	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477	{
1478	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1479	#if HC_ARCH_BITS == 64
1480	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1481	{
1482	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1483	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1484	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1485	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1486	}
1487	#endif
1488	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1489	}
1490	}
1491
1492
1493	/**
1494	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1495	* lazily while leaving VT-x.
1496	*
1497	* @returns true if it does, false otherwise.
1498	* @param pVCpu The cross context virtual CPU structure.
1499	* @param uMsr The MSR to check.
1500	*/
1501	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1502	{
1503	NOREF(pVCpu);
1504	#if HC_ARCH_BITS == 64
1505	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1506	{
1507	switch (uMsr)
1508	{
1509	case MSR_K8_LSTAR:
1510	case MSR_K6_STAR:
1511	case MSR_K8_SF_MASK:
1512	case MSR_K8_KERNEL_GS_BASE:
1513	return true;
1514	}
1515	}
1516	#else
1517	RT_NOREF(pVCpu, uMsr);
1518	#endif
1519	return false;
1520	}
1521
1522
1523	/**
1524	* Saves a set of guest MSRs back into the guest-CPU context.
1525	*
1526	* @param pVCpu The cross context virtual CPU structure.
1527	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1528	* out-of-sync. Make sure to update the required fields
1529	* before using them.
1530	*
1531	* @remarks No-long-jump zone!!!
1532	*/
1533	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1534	{
1535	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1536	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1537
1538	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1539	{
1540	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1541	#if HC_ARCH_BITS == 64
1542	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1543	{
1544	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1545	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1546	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1547	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1548	}
1549	#else
1550	NOREF(pMixedCtx);
1551	#endif
1552	}
1553	}
1554
1555
1556	/**
1557	* Loads a set of guests MSRs to allow read/passthru to the guest.
1558	*
1559	* The name of this function is slightly confusing. This function does NOT
1560	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1561	* common prefix for functions dealing with "lazy restoration" of the shared
1562	* MSRs.
1563	*
1564	* @param pVCpu The cross context virtual CPU structure.
1565	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1566	* out-of-sync. Make sure to update the required fields
1567	* before using them.
1568	*
1569	* @remarks No-long-jump zone!!!
1570	*/
1571	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1572	{
1573	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1574	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1575
1576	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1577	#if HC_ARCH_BITS == 64
1578	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1579	{
1580	/*
1581	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1582	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1583	* we can skip a few MSR writes.
1584	*
1585	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1586	* guest MSR values in the guest-CPU context might be different to what's currently
1587	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1588	* CPU, see @bugref{8728}.
1589	*/
1590	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1591	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1592	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1593	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1594	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1595	{
1596	#ifdef VBOX_STRICT
1597	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1598	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1599	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1600	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1601	#endif
1602	}
1603	else
1604	{
1605	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1606	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1607	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1608	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1609	}
1610	}
1611	#else
1612	RT_NOREF(pMixedCtx);
1613	#endif
1614	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1615	}
1616
1617
1618	/**
1619	* Performs lazy restoration of the set of host MSRs if they were previously
1620	* loaded with guest MSR values.
1621	*
1622	* @param pVCpu The cross context virtual CPU structure.
1623	*
1624	* @remarks No-long-jump zone!!!
1625	* @remarks The guest MSRs should have been saved back into the guest-CPU
1626	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1627	*/
1628	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1629	{
1630	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1631	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1632
1633	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1634	{
1635	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1636	#if HC_ARCH_BITS == 64
1637	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1638	{
1639	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1640	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1641	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1642	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1643	}
1644	#endif
1645	}
1646	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1647	}
1648
1649
1650	/**
1651	* Verifies that our cached values of the VMCS controls are all
1652	* consistent with what's actually present in the VMCS.
1653	*
1654	* @returns VBox status code.
1655	* @param pVCpu The cross context virtual CPU structure.
1656	*/
1657	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1658	{
1659	uint32_t u32Val;
1660	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1661	AssertRCReturn(rc, rc);
1662	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1663	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1664
1665	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1666	AssertRCReturn(rc, rc);
1667	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1668	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1669
1670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1671	AssertRCReturn(rc, rc);
1672	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1673	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1674
1675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1676	AssertRCReturn(rc, rc);
1677	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1678	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1679
1680	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1681	{
1682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1683	AssertRCReturn(rc, rc);
1684	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1685	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1686	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1687	}
1688
1689	return VINF_SUCCESS;
1690	}
1691
1692
1693	#ifdef VBOX_STRICT
1694	/**
1695	* Verifies that our cached host EFER value has not changed
1696	* since we cached it.
1697	*
1698	* @param pVCpu The cross context virtual CPU structure.
1699	*/
1700	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1701	{
1702	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1703
1704	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1705	{
1706	uint64_t u64Val;
1707	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1708	AssertRC(rc);
1709
1710	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1711	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1712	}
1713	}
1714
1715
1716	/**
1717	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1718	* VMCS are correct.
1719	*
1720	* @param pVCpu The cross context virtual CPU structure.
1721	*/
1722	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1723	{
1724	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1725
1726	/* Verify MSR counts in the VMCS are what we think it should be. */
1727	uint32_t cMsrs;
1728	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1729	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1730
1731	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1732	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1733
1734	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1735	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1736
1737	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1738	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1739	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1740	{
1741	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1742	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1743	pGuestMsr->u32Msr, cMsrs));
1744
1745	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1746	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1747	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1748
1749	/* Verify that the permissions are as expected in the MSR bitmap. */
1750	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1751	{
1752	VMXMSREXITREAD enmRead;
1753	VMXMSREXITWRITE enmWrite;
1754	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1755	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1756	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1757	{
1758	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1759	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1760	}
1761	else
1762	{
1763	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1764	pGuestMsr->u32Msr, cMsrs));
1765	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1766	pGuestMsr->u32Msr, cMsrs));
1767	}
1768	}
1769	}
1770	}
1771	#endif /* VBOX_STRICT */
1772
1773
1774	/**
1775	* Flushes the TLB using EPT.
1776	*
1777	* @returns VBox status code.
1778	* @param pVCpu The cross context virtual CPU structure of the calling
1779	* EMT. Can be NULL depending on @a enmFlush.
1780	* @param enmFlush Type of flush.
1781	*
1782	* @remarks Caller is responsible for making sure this function is called only
1783	* when NestedPaging is supported and providing @a enmFlush that is
1784	* supported by the CPU.
1785	* @remarks Can be called with interrupts disabled.
1786	*/
1787	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1788	{
1789	uint64_t au64Descriptor[2];
1790	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1791	au64Descriptor[0] = 0;
1792	else
1793	{
1794	Assert(pVCpu);
1795	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1796	}
1797	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1798
1799	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1800	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1801	rc));
1802	if ( RT_SUCCESS(rc)
1803	&& pVCpu)
1804	{
1805	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1806	}
1807	}
1808
1809
1810	/**
1811	* Flushes the TLB using VPID.
1812	*
1813	* @returns VBox status code.
1814	* @param pVM The cross context VM structure.
1815	* @param pVCpu The cross context virtual CPU structure of the calling
1816	* EMT. Can be NULL depending on @a enmFlush.
1817	* @param enmFlush Type of flush.
1818	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1819	* on @a enmFlush).
1820	*
1821	* @remarks Can be called with interrupts disabled.
1822	*/
1823	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1824	{
1825	NOREF(pVM);
1826	AssertPtr(pVM);
1827	Assert(pVM->hm.s.vmx.fVpid);
1828
1829	uint64_t au64Descriptor[2];
1830	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1831	{
1832	au64Descriptor[0] = 0;
1833	au64Descriptor[1] = 0;
1834	}
1835	else
1836	{
1837	AssertPtr(pVCpu);
1838	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1839	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1840	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1841	au64Descriptor[1] = GCPtr;
1842	}
1843
1844	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1845	AssertMsg(rc == VINF_SUCCESS,
1846	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1847	if ( RT_SUCCESS(rc)
1848	&& pVCpu)
1849	{
1850	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1851	}
1852	}
1853
1854
1855	/**
1856	* Invalidates a guest page by guest virtual address. Only relevant for
1857	* EPT/VPID, otherwise there is nothing really to invalidate.
1858	*
1859	* @returns VBox status code.
1860	* @param pVM The cross context VM structure.
1861	* @param pVCpu The cross context virtual CPU structure.
1862	* @param GCVirt Guest virtual address of the page to invalidate.
1863	*/
1864	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1865	{
1866	AssertPtr(pVM);
1867	AssertPtr(pVCpu);
1868	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1869
1870	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1871	if (!fFlushPending)
1872	{
1873	/*
1874	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1875	* See @bugref{6043} and @bugref{6177}.
1876	*
1877	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1878	* function maybe called in a loop with individual addresses.
1879	*/
1880	if (pVM->hm.s.vmx.fVpid)
1881	{
1882	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1883	{
1884	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1885	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1886	}
1887	else
1888	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889	}
1890	else if (pVM->hm.s.fNestedPaging)
1891	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1892	}
1893
1894	return VINF_SUCCESS;
1895	}
1896
1897
1898	/**
1899	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1900	* otherwise there is nothing really to invalidate.
1901	*
1902	* @returns VBox status code.
1903	* @param pVM The cross context VM structure.
1904	* @param pVCpu The cross context virtual CPU structure.
1905	* @param GCPhys Guest physical address of the page to invalidate.
1906	*/
1907	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1908	{
1909	NOREF(pVM); NOREF(GCPhys);
1910	LogFlowFunc(("%RGp\n", GCPhys));
1911
1912	/*
1913	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1914	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1915	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1916	*/
1917	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1918	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1919	return VINF_SUCCESS;
1920	}
1921
1922
1923	/**
1924	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1925	* case where neither EPT nor VPID is supported by the CPU.
1926	*
1927	* @param pVM The cross context VM structure.
1928	* @param pVCpu The cross context virtual CPU structure.
1929	* @param pCpu Pointer to the global HM struct.
1930	*
1931	* @remarks Called with interrupts disabled.
1932	*/
1933	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1934	{
1935	AssertPtr(pVCpu);
1936	AssertPtr(pCpu);
1937	NOREF(pVM);
1938
1939	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1940
1941	Assert(pCpu->idCpu != NIL_RTCPUID);
1942	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1943	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1944	pVCpu->hm.s.fForceTLBFlush = false;
1945	return;
1946	}
1947
1948
1949	/**
1950	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1951	*
1952	* @param pVM The cross context VM structure.
1953	* @param pVCpu The cross context virtual CPU structure.
1954	* @param pCpu Pointer to the global HM CPU struct.
1955	* @remarks All references to "ASID" in this function pertains to "VPID" in
1956	* Intel's nomenclature. The reason is, to avoid confusion in compare
1957	* statements since the host-CPU copies are named "ASID".
1958	*
1959	* @remarks Called with interrupts disabled.
1960	*/
1961	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1962	{
1963	#ifdef VBOX_WITH_STATISTICS
1964	bool fTlbFlushed = false;
1965	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1966	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1967	if (!fTlbFlushed) \
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1969	} while (0)
1970	#else
1971	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1972	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1973	#endif
1974
1975	AssertPtr(pVM);
1976	AssertPtr(pCpu);
1977	AssertPtr(pVCpu);
1978	Assert(pCpu->idCpu != NIL_RTCPUID);
1979
1980	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1981	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1982	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1983
1984	/*
1985	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1986	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1987	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1988	*/
1989	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1990	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1991	{
1992	++pCpu->uCurrentAsid;
1993	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1994	{
1995	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1996	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1997	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1998	}
1999
2000	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2001	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2002	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2003
2004	/*
2005	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2006	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2007	*/
2008	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2009	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2010	HMVMX_SET_TAGGED_TLB_FLUSHED();
2011	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2012	}
2013
2014	/* Check for explicit TLB flushes. */
2015	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2016	{
2017	/*
2018	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2019	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2020	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2021	* but not guest-physical mappings.
2022	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2023	*/
2024	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2025	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2026	HMVMX_SET_TAGGED_TLB_FLUSHED();
2027	}
2028
2029	pVCpu->hm.s.fForceTLBFlush = false;
2030	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2031
2032	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2033	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2034	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2035	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2036	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2037	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2038	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2039	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2040	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2041
2042	/* Update VMCS with the VPID. */
2043	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2044	AssertRC(rc);
2045
2046	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2047	}
2048
2049
2050	/**
2051	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2052	*
2053	* @returns VBox status code.
2054	* @param pVM The cross context VM structure.
2055	* @param pVCpu The cross context virtual CPU structure.
2056	* @param pCpu Pointer to the global HM CPU struct.
2057	*
2058	* @remarks Called with interrupts disabled.
2059	*/
2060	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2061	{
2062	AssertPtr(pVM);
2063	AssertPtr(pVCpu);
2064	AssertPtr(pCpu);
2065	Assert(pCpu->idCpu != NIL_RTCPUID);
2066	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2067	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2068
2069	/*
2070	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2071	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2072	*/
2073	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2074	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2075	{
2076	pVCpu->hm.s.fForceTLBFlush = true;
2077	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2078	}
2079
2080	/* Check for explicit TLB flushes. */
2081	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2082	{
2083	pVCpu->hm.s.fForceTLBFlush = true;
2084	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2085	}
2086
2087	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2088	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2089
2090	if (pVCpu->hm.s.fForceTLBFlush)
2091	{
2092	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2093	pVCpu->hm.s.fForceTLBFlush = false;
2094	}
2095	}
2096
2097
2098	/**
2099	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2100	*
2101	* @returns VBox status code.
2102	* @param pVM The cross context VM structure.
2103	* @param pVCpu The cross context virtual CPU structure.
2104	* @param pCpu Pointer to the global HM CPU struct.
2105	*
2106	* @remarks Called with interrupts disabled.
2107	*/
2108	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2109	{
2110	AssertPtr(pVM);
2111	AssertPtr(pVCpu);
2112	AssertPtr(pCpu);
2113	Assert(pCpu->idCpu != NIL_RTCPUID);
2114	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2115	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2116
2117	/*
2118	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2119	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2120	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2121	*/
2122	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2123	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2124	{
2125	pVCpu->hm.s.fForceTLBFlush = true;
2126	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2127	}
2128
2129	/* Check for explicit TLB flushes. */
2130	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2131	{
2132	/*
2133	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2134	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2135	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2136	*/
2137	pVCpu->hm.s.fForceTLBFlush = true;
2138	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2139	}
2140
2141	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2142	if (pVCpu->hm.s.fForceTLBFlush)
2143	{
2144	++pCpu->uCurrentAsid;
2145	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2146	{
2147	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2148	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2149	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2150	}
2151
2152	pVCpu->hm.s.fForceTLBFlush = false;
2153	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2154	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2155	if (pCpu->fFlushAsidBeforeUse)
2156	{
2157	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2158	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2159	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2160	{
2161	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2162	pCpu->fFlushAsidBeforeUse = false;
2163	}
2164	else
2165	{
2166	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2167	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2168	}
2169	}
2170	}
2171
2172	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2173	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2174	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2175	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2176	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2177	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2178	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2179
2180	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2181	AssertRC(rc);
2182	}
2183
2184
2185	/**
2186	* Flushes the guest TLB entry based on CPU capabilities.
2187	*
2188	* @param pVCpu The cross context virtual CPU structure.
2189	* @param pCpu Pointer to the global HM CPU struct.
2190	*/
2191	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2192	{
2193	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2194	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2195	#endif
2196	PVM pVM = pVCpu->CTX_SUFF(pVM);
2197	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2198	{
2199	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2200	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2201	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2202	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2203	default:
2204	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2205	break;
2206	}
2207
2208	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2209	}
2210
2211
2212	/**
2213	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2214	* TLB entries from the host TLB before VM-entry.
2215	*
2216	* @returns VBox status code.
2217	* @param pVM The cross context VM structure.
2218	*/
2219	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2220	{
2221	/*
2222	* Determine optimal flush type for Nested Paging.
2223	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2224	* guest execution (see hmR3InitFinalizeR0()).
2225	*/
2226	if (pVM->hm.s.fNestedPaging)
2227	{
2228	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2229	{
2230	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2231	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2232	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2233	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2234	else
2235	{
2236	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2237	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2238	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2239	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2240	}
2241
2242	/* Make sure the write-back cacheable memory type for EPT is supported. */
2243	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2244	{
2245	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2246	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2247	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2248	}
2249
2250	/* EPT requires a page-walk length of 4. */
2251	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2252	{
2253	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2254	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2255	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2256	}
2257	}
2258	else
2259	{
2260	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2261	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2263	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264	}
2265	}
2266
2267	/*
2268	* Determine optimal flush type for VPID.
2269	*/
2270	if (pVM->hm.s.vmx.fVpid)
2271	{
2272	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2273	{
2274	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2275	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2276	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2277	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2278	else
2279	{
2280	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2281	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2282	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2283	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2284	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2285	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2286	pVM->hm.s.vmx.fVpid = false;
2287	}
2288	}
2289	else
2290	{
2291	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2292	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2293	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2294	pVM->hm.s.vmx.fVpid = false;
2295	}
2296	}
2297
2298	/*
2299	* Setup the handler for flushing tagged-TLBs.
2300	*/
2301	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2302	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2303	else if (pVM->hm.s.fNestedPaging)
2304	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2305	else if (pVM->hm.s.vmx.fVpid)
2306	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2307	else
2308	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2309	return VINF_SUCCESS;
2310	}
2311
2312
2313	/**
2314	* Sets up pin-based VM-execution controls in the VMCS.
2315	*
2316	* @returns VBox status code.
2317	* @param pVM The cross context VM structure.
2318	* @param pVCpu The cross context virtual CPU structure.
2319	*/
2320	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2321	{
2322	AssertPtr(pVM);
2323	AssertPtr(pVCpu);
2324
2325	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2326	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2327
2328	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2329	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2330
2331	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2332	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2333
2334	/* Enable the VMX preemption timer. */
2335	if (pVM->hm.s.vmx.fUsePreemptTimer)
2336	{
2337	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2338	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2339	}
2340
2341	#if 0
2342	/* Enable posted-interrupt processing. */
2343	if (pVM->hm.s.fPostedIntrs)
2344	{
2345	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2346	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2347	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2348	}
2349	#endif
2350
2351	if ((val & zap) != val)
2352	{
2353	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2354	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2355	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2356	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2357	}
2358
2359	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2360	AssertRCReturn(rc, rc);
2361
2362	pVCpu->hm.s.vmx.u32PinCtls = val;
2363	return rc;
2364	}
2365
2366
2367	/**
2368	* Sets up processor-based VM-execution controls in the VMCS.
2369	*
2370	* @returns VBox status code.
2371	* @param pVM The cross context VM structure.
2372	* @param pVCpu The cross context virtual CPU structure.
2373	*/
2374	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2375	{
2376	AssertPtr(pVM);
2377	AssertPtr(pVCpu);
2378
2379	int rc = VERR_INTERNAL_ERROR_5;
2380	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2381	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2382
2383	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2384	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2385	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2386	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2387	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2388	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2389	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2390
2391	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2392	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2393	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2394	{
2395	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2396	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2397	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2398	}
2399
2400	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2401	if (!pVM->hm.s.fNestedPaging)
2402	{
2403	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2404	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2405	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2406	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2407	}
2408
2409	/* Use TPR shadowing if supported by the CPU. */
2410	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2411	{
2412	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2413	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2414	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2415	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2416	AssertRCReturn(rc, rc);
2417
2418	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2419	/* CR8 writes cause a VM-exit based on TPR threshold. */
2420	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2421	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2422	}
2423	else
2424	{
2425	/*
2426	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2427	* Set this control only for 64-bit guests.
2428	*/
2429	if (pVM->hm.s.fAllow64BitGuests)
2430	{
2431	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2432	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2433	}
2434	}
2435
2436	/* Use MSR-bitmaps if supported by the CPU. */
2437	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2438	{
2439	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2440
2441	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2442	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2443	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2444	AssertRCReturn(rc, rc);
2445
2446	/*
2447	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2448	* automatically using dedicated fields in the VMCS.
2449	*/
2450	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2455
2456	#if HC_ARCH_BITS == 64
2457	/*
2458	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2459	*/
2460	if (pVM->hm.s.fAllow64BitGuests)
2461	{
2462	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2463	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2464	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2465	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2466	}
2467	#endif
2468	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2469	}
2470
2471	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2472	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2473	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2474
2475	if ((val & zap) != val)
2476	{
2477	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2478	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2479	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2480	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2481	}
2482
2483	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2484	AssertRCReturn(rc, rc);
2485
2486	pVCpu->hm.s.vmx.u32ProcCtls = val;
2487
2488	/*
2489	* Secondary processor-based VM-execution controls.
2490	*/
2491	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2492	{
2493	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2494	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2495
2496	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2497	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2498
2499	if (pVM->hm.s.fNestedPaging)
2500	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2501	else
2502	{
2503	/*
2504	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2505	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2506	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2507	*/
2508	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2509	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2510	}
2511
2512	if (pVM->hm.s.vmx.fVpid)
2513	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2514
2515	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2516	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2517
2518	#if 0
2519	if (pVM->hm.s.fVirtApicRegs)
2520	{
2521	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2522	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2523
2524	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2525	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2526	}
2527	#endif
2528
2529	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2530	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2531	* done dynamically. */
2532	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2533	{
2534	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2535	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2536	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2537	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2538	AssertRCReturn(rc, rc);
2539	}
2540
2541	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2542	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2543
2544	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2545	&& pVM->hm.s.vmx.cPleGapTicks
2546	&& pVM->hm.s.vmx.cPleWindowTicks)
2547	{
2548	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2549
2550	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2551	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2552	AssertRCReturn(rc, rc);
2553	}
2554
2555	if ((val & zap) != val)
2556	{
2557	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2558	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2559	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2560	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2561	}
2562
2563	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2564	AssertRCReturn(rc, rc);
2565
2566	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2567	}
2568	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2569	{
2570	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2571	"available\n"));
2572	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2573	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2574	}
2575
2576	return VINF_SUCCESS;
2577	}
2578
2579
2580	/**
2581	* Sets up miscellaneous (everything other than Pin & Processor-based
2582	* VM-execution) control fields in the VMCS.
2583	*
2584	* @returns VBox status code.
2585	* @param pVM The cross context VM structure.
2586	* @param pVCpu The cross context virtual CPU structure.
2587	*/
2588	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2589	{
2590	NOREF(pVM);
2591	AssertPtr(pVM);
2592	AssertPtr(pVCpu);
2593
2594	int rc = VERR_GENERAL_FAILURE;
2595
2596	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2597	#if 0
2598	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2599	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2600	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2601
2602	/*
2603	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2604	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2605	* We thus use the exception bitmap to control it rather than use both.
2606	*/
2607	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2608	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2609
2610	/** @todo Explore possibility of using IO-bitmaps. */
2611	/* All IO & IOIO instructions cause VM-exits. */
2612	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2613	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2614
2615	/* Initialize the MSR-bitmap area. */
2616	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2617	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2618	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2619	AssertRCReturn(rc, rc);
2620	#endif
2621
2622	/* Setup MSR auto-load/store area. */
2623	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2624	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2625	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2626	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2627	AssertRCReturn(rc, rc);
2628
2629	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2630	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2631	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2632	AssertRCReturn(rc, rc);
2633
2634	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2635	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2636	AssertRCReturn(rc, rc);
2637
2638	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2639	#if 0
2640	/* Setup debug controls */
2641	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2642	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2643	AssertRCReturn(rc, rc);
2644	#endif
2645
2646	return rc;
2647	}
2648
2649
2650	/**
2651	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2652	*
2653	* We shall setup those exception intercepts that don't change during the
2654	* lifetime of the VM here. The rest are done dynamically while loading the
2655	* guest state.
2656	*
2657	* @returns VBox status code.
2658	* @param pVM The cross context VM structure.
2659	* @param pVCpu The cross context virtual CPU structure.
2660	*/
2661	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2662	{
2663	AssertPtr(pVM);
2664	AssertPtr(pVCpu);
2665
2666	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2667
2668	uint32_t u32XcptBitmap = 0;
2669
2670	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2671	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2672
2673	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2674	and writes, and because recursive #DBs can cause the CPU hang, we must always
2675	intercept #DB. */
2676	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2677
2678	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2679	if (!pVM->hm.s.fNestedPaging)
2680	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2681
2682	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2683	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2684	AssertRCReturn(rc, rc);
2685	return rc;
2686	}
2687
2688
2689	/**
2690	* Sets up the initial guest-state mask. The guest-state mask is consulted
2691	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2692	* for the nested virtualization case (as it would cause a VM-exit).
2693	*
2694	* @param pVCpu The cross context virtual CPU structure.
2695	*/
2696	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2697	{
2698	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2699	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2700	return VINF_SUCCESS;
2701	}
2702
2703
2704	/**
2705	* Does per-VM VT-x initialization.
2706	*
2707	* @returns VBox status code.
2708	* @param pVM The cross context VM structure.
2709	*/
2710	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2711	{
2712	LogFlowFunc(("pVM=%p\n", pVM));
2713
2714	int rc = hmR0VmxStructsAlloc(pVM);
2715	if (RT_FAILURE(rc))
2716	{
2717	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2718	return rc;
2719	}
2720
2721	return VINF_SUCCESS;
2722	}
2723
2724
2725	/**
2726	* Does per-VM VT-x termination.
2727	*
2728	* @returns VBox status code.
2729	* @param pVM The cross context VM structure.
2730	*/
2731	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2732	{
2733	LogFlowFunc(("pVM=%p\n", pVM));
2734
2735	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2736	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2737	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2738	#endif
2739	hmR0VmxStructsFree(pVM);
2740	return VINF_SUCCESS;
2741	}
2742
2743
2744	/**
2745	* Sets up the VM for execution under VT-x.
2746	* This function is only called once per-VM during initialization.
2747	*
2748	* @returns VBox status code.
2749	* @param pVM The cross context VM structure.
2750	*/
2751	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2752	{
2753	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2754	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2755
2756	LogFlowFunc(("pVM=%p\n", pVM));
2757
2758	/*
2759	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2760	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2761	*/
2762	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2763	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2764	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2765	{
2766	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2767	return VERR_INTERNAL_ERROR;
2768	}
2769
2770	/* Initialize these always, see hmR3InitFinalizeR0().*/
2771	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2772	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2773
2774	/* Setup the tagged-TLB flush handlers. */
2775	int rc = hmR0VmxSetupTaggedTlb(pVM);
2776	if (RT_FAILURE(rc))
2777	{
2778	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2779	return rc;
2780	}
2781
2782	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2783	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2784	#if HC_ARCH_BITS == 64
2785	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2786	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2787	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2788	{
2789	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2790	}
2791	#endif
2792
2793	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2794	RTCCUINTREG uHostCR4 = ASMGetCR4();
2795	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2796	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2797
2798	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2799	{
2800	PVMCPU pVCpu = &pVM->aCpus[i];
2801	AssertPtr(pVCpu);
2802	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2803
2804	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2805	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2806
2807	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2808	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2809	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2810
2811	/* Set revision dword at the beginning of the VMCS structure. */
2812	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2813
2814	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2815	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2816	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818
2819	/* Load this VMCS as the current VMCS. */
2820	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2821	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2825	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2826	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2827
2828	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2829	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2830	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2831
2832	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2833	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2834	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2835
2836	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2837	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2838	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2839
2840	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2841	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2842	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2843
2844	#if HC_ARCH_BITS == 32
2845	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2846	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2847	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2848	#endif
2849
2850	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2851	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2852	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2853	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2854
2855	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2856
2857	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2858	}
2859
2860	return VINF_SUCCESS;
2861	}
2862
2863
2864	/**
2865	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2866	* the VMCS.
2867	*
2868	* @returns VBox status code.
2869	* @param pVM The cross context VM structure.
2870	* @param pVCpu The cross context virtual CPU structure.
2871	*/
2872	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2873	{
2874	NOREF(pVM); NOREF(pVCpu);
2875
2876	RTCCUINTREG uReg = ASMGetCR0();
2877	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2878	AssertRCReturn(rc, rc);
2879
2880	uReg = ASMGetCR3();
2881	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2882	AssertRCReturn(rc, rc);
2883
2884	uReg = ASMGetCR4();
2885	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2886	AssertRCReturn(rc, rc);
2887	return rc;
2888	}
2889
2890
2891	#if HC_ARCH_BITS == 64
2892	/**
2893	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2894	* requirements. See hmR0VmxSaveHostSegmentRegs().
2895	*/
2896	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2897	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2898	{ \
2899	bool fValidSelector = true; \
2900	if ((selValue) & X86_SEL_LDT) \
2901	{ \
2902	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2903	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2904	} \
2905	if (fValidSelector) \
2906	{ \
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2908	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2909	} \
2910	(selValue) = 0; \
2911	}
2912	#endif
2913
2914
2915	/**
2916	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2917	* the host-state area in the VMCS.
2918	*
2919	* @returns VBox status code.
2920	* @param pVM The cross context VM structure.
2921	* @param pVCpu The cross context virtual CPU structure.
2922	*/
2923	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2924	{
2925	int rc = VERR_INTERNAL_ERROR_5;
2926
2927	#if HC_ARCH_BITS == 64
2928	/*
2929	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2930	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2931	*
2932	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2933	* Was observed booting Solaris10u10 32-bit guest.
2934	*/
2935	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2936	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2937	{
2938	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2939	pVCpu->idCpu));
2940	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2941	}
2942	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2943	#else
2944	RT_NOREF(pVCpu);
2945	#endif
2946
2947	/*
2948	* Host DS, ES, FS and GS segment registers.
2949	*/
2950	#if HC_ARCH_BITS == 64
2951	RTSEL uSelDS = ASMGetDS();
2952	RTSEL uSelES = ASMGetES();
2953	RTSEL uSelFS = ASMGetFS();
2954	RTSEL uSelGS = ASMGetGS();
2955	#else
2956	RTSEL uSelDS = 0;
2957	RTSEL uSelES = 0;
2958	RTSEL uSelFS = 0;
2959	RTSEL uSelGS = 0;
2960	#endif
2961
2962	/*
2963	* Host CS and SS segment registers.
2964	*/
2965	RTSEL uSelCS = ASMGetCS();
2966	RTSEL uSelSS = ASMGetSS();
2967
2968	/*
2969	* Host TR segment register.
2970	*/
2971	RTSEL uSelTR = ASMGetTR();
2972
2973	#if HC_ARCH_BITS == 64
2974	/*
2975	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2976	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2977	*/
2978	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2979	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2980	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2981	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2982	# undef VMXLOCAL_ADJUST_HOST_SEG
2983	#endif
2984
2985	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2986	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2987	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2988	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2989	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2990	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2991	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2992	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2993	Assert(uSelCS);
2994	Assert(uSelTR);
2995
2996	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2997	#if 0
2998	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2999	Assert(uSelSS != 0);
3000	#endif
3001
3002	/* Write these host selector fields into the host-state area in the VMCS. */
3003	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3004	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3005	#if HC_ARCH_BITS == 64
3006	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3007	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3008	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3009	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3010	#else
3011	NOREF(uSelDS);
3012	NOREF(uSelES);
3013	NOREF(uSelFS);
3014	NOREF(uSelGS);
3015	#endif
3016	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3017	AssertRCReturn(rc, rc);
3018
3019	/*
3020	* Host GDTR and IDTR.
3021	*/
3022	RTGDTR Gdtr;
3023	RTIDTR Idtr;
3024	RT_ZERO(Gdtr);
3025	RT_ZERO(Idtr);
3026	ASMGetGDTR(&Gdtr);
3027	ASMGetIDTR(&Idtr);
3028	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3029	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3030	AssertRCReturn(rc, rc);
3031
3032	#if HC_ARCH_BITS == 64
3033	/*
3034	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3035	* maximum limit (0xffff) on every VM-exit.
3036	*/
3037	if (Gdtr.cbGdt != 0xffff)
3038	{
3039	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3040	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3041	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3042	}
3043
3044	/*
3045	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3046	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3047	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3048	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3049	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3050	* hosts where we are pretty sure it won't cause trouble.
3051	*/
3052	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3053	if (Idtr.cbIdt < 0x0fff)
3054	# else
3055	if (Idtr.cbIdt != 0xffff)
3056	# endif
3057	{
3058	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3059	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3060	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3061	}
3062	#endif
3063
3064	/*
3065	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3066	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3067	*/
3068	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3069	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3070	VERR_VMX_INVALID_HOST_STATE);
3071
3072	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3073	#if HC_ARCH_BITS == 64
3074	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3075
3076	/*
3077	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3078	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3079	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3080	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3081	*
3082	* [1] See Intel spec. 3.5 "System Descriptor Types".
3083	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3084	*/
3085	Assert(pDesc->System.u4Type == 11);
3086	if ( pDesc->System.u16LimitLow != 0x67
3087	\|\| pDesc->System.u4LimitHigh)
3088	{
3089	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3090	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3091	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3092	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3093	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3094
3095	/* Store the GDTR here as we need it while restoring TR. */
3096	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3097	}
3098	#else
3099	NOREF(pVM);
3100	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3101	#endif
3102	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3103	AssertRCReturn(rc, rc);
3104
3105	/*
3106	* Host FS base and GS base.
3107	*/
3108	#if HC_ARCH_BITS == 64
3109	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3110	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3111	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3112	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3113	AssertRCReturn(rc, rc);
3114
3115	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3116	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3117	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3118	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3119	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3120	#endif
3121	return rc;
3122	}
3123
3124
3125	/**
3126	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3127	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3128	* the host after every successful VM-exit.
3129	*
3130	* @returns VBox status code.
3131	* @param pVM The cross context VM structure.
3132	* @param pVCpu The cross context virtual CPU structure.
3133	*
3134	* @remarks No-long-jump zone!!!
3135	*/
3136	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3137	{
3138	NOREF(pVM);
3139
3140	AssertPtr(pVCpu);
3141	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3142
3143	/*
3144	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3145	* rather than swapping them on every VM-entry.
3146	*/
3147	hmR0VmxLazySaveHostMsrs(pVCpu);
3148
3149	/*
3150	* Host Sysenter MSRs.
3151	*/
3152	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3153	#if HC_ARCH_BITS == 32
3154	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3155	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3156	#else
3157	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3158	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3159	#endif
3160	AssertRCReturn(rc, rc);
3161
3162	/*
3163	* Host EFER MSR.
3164	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3165	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3166	*/
3167	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3168	{
3169	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3170	AssertRCReturn(rc, rc);
3171	}
3172
3173	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3174	* hmR0VmxLoadGuestExitCtls() !! */
3175
3176	return rc;
3177	}
3178
3179
3180	/**
3181	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3182	*
3183	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3184	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3185	* hmR0VMxLoadGuestEntryCtls().
3186	*
3187	* @returns true if we need to load guest EFER, false otherwise.
3188	* @param pVCpu The cross context virtual CPU structure.
3189	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3190	* out-of-sync. Make sure to update the required fields
3191	* before using them.
3192	*
3193	* @remarks Requires EFER, CR4.
3194	* @remarks No-long-jump zone!!!
3195	*/
3196	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3197	{
3198	#ifdef HMVMX_ALWAYS_SWAP_EFER
3199	return true;
3200	#endif
3201
3202	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3203	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3204	if (CPUMIsGuestInLongMode(pVCpu))
3205	return false;
3206	#endif
3207
3208	PVM pVM = pVCpu->CTX_SUFF(pVM);
3209	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3210	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3211
3212	/*
3213	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3214	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3215	*/
3216	if ( CPUMIsGuestInLongMode(pVCpu)
3217	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3218	{
3219	return true;
3220	}
3221
3222	/*
3223	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3224	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3225	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3226	*/
3227	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3228	&& (pMixedCtx->cr0 & X86_CR0_PG)
3229	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3230	{
3231	/* Assert that host is PAE capable. */
3232	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3233	return true;
3234	}
3235
3236	/** @todo Check the latest Intel spec. for any other bits,
3237	* like SMEP/SMAP? */
3238	return false;
3239	}
3240
3241
3242	/**
3243	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3244	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3245	* controls".
3246	*
3247	* @returns VBox status code.
3248	* @param pVCpu The cross context virtual CPU structure.
3249	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3250	* out-of-sync. Make sure to update the required fields
3251	* before using them.
3252	*
3253	* @remarks Requires EFER.
3254	* @remarks No-long-jump zone!!!
3255	*/
3256	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3257	{
3258	int rc = VINF_SUCCESS;
3259	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3260	{
3261	PVM pVM = pVCpu->CTX_SUFF(pVM);
3262	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3263	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3264
3265	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3266	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3267
3268	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3269	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3270	{
3271	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3272	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3273	}
3274	else
3275	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3276
3277	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3278	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3279	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3280	{
3281	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3282	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3283	}
3284
3285	/*
3286	* The following should -not- be set (since we're not in SMM mode):
3287	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3288	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3289	*/
3290
3291	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3292	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3293
3294	if ((val & zap) != val)
3295	{
3296	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3297	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3298	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3299	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3300	}
3301
3302	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3303	AssertRCReturn(rc, rc);
3304
3305	pVCpu->hm.s.vmx.u32EntryCtls = val;
3306	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3307	}
3308	return rc;
3309	}
3310
3311
3312	/**
3313	* Sets up the VM-exit controls in the VMCS.
3314	*
3315	* @returns VBox status code.
3316	* @param pVCpu The cross context virtual CPU structure.
3317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3318	* out-of-sync. Make sure to update the required fields
3319	* before using them.
3320	*
3321	* @remarks Requires EFER.
3322	*/
3323	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3324	{
3325	NOREF(pMixedCtx);
3326
3327	int rc = VINF_SUCCESS;
3328	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3329	{
3330	PVM pVM = pVCpu->CTX_SUFF(pVM);
3331	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3332	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3333
3334	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3335	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3336
3337	/*
3338	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3339	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3340	*/
3341	#if HC_ARCH_BITS == 64
3342	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3343	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3344	#else
3345	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3346	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3347	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3348	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3349	{
3350	/* The switcher returns to long mode, EFER is managed by the switcher. */
3351	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3352	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3353	}
3354	else
3355	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3356	#endif
3357
3358	/* If the newer VMCS fields for managing EFER exists, use it. */
3359	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3360	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3361	{
3362	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3363	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3364	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3365	}
3366
3367	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3368	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3369
3370	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3371	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3372	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3373
3374	if ( pVM->hm.s.vmx.fUsePreemptTimer
3375	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3376	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3377
3378	if ((val & zap) != val)
3379	{
3380	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3381	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3382	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3383	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3384	}
3385
3386	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3387	AssertRCReturn(rc, rc);
3388
3389	pVCpu->hm.s.vmx.u32ExitCtls = val;
3390	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3391	}
3392	return rc;
3393	}
3394
3395
3396	/**
3397	* Sets the TPR threshold in the VMCS.
3398	*
3399	* @returns VBox status code.
3400	* @param pVCpu The cross context virtual CPU structure.
3401	* @param u32TprThreshold The TPR threshold (task-priority class only).
3402	*/
3403	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3404	{
3405	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3406	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3407	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3408	}
3409
3410
3411	/**
3412	* Loads the guest APIC and related state.
3413	*
3414	* @returns VBox status code.
3415	* @param pVCpu The cross context virtual CPU structure.
3416	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3417	* out-of-sync. Make sure to update the required fields
3418	* before using them.
3419	*
3420	* @remarks No-long-jump zone!!!
3421	*/
3422	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3423	{
3424	NOREF(pMixedCtx);
3425
3426	int rc = VINF_SUCCESS;
3427	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3428	{
3429	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3430	&& APICIsEnabled(pVCpu))
3431	{
3432	/*
3433	* Setup TPR shadowing.
3434	*/
3435	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3436	{
3437	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3438
3439	bool fPendingIntr = false;
3440	uint8_t u8Tpr = 0;
3441	uint8_t u8PendingIntr = 0;
3442	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3443	AssertRCReturn(rc, rc);
3444
3445	/*
3446	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3447	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3448	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3449	*/
3450	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3451	uint32_t u32TprThreshold = 0;
3452	if (fPendingIntr)
3453	{
3454	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3455	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3456	const uint8_t u8TprPriority = u8Tpr >> 4;
3457	if (u8PendingPriority <= u8TprPriority)
3458	u32TprThreshold = u8PendingPriority;
3459	}
3460
3461	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3462	AssertRCReturn(rc, rc);
3463	}
3464	}
3465	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3466	}
3467
3468	return rc;
3469	}
3470
3471
3472	/**
3473	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3474	*
3475	* @returns Guest's interruptibility-state.
3476	* @param pVCpu The cross context virtual CPU structure.
3477	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3478	* out-of-sync. Make sure to update the required fields
3479	* before using them.
3480	*
3481	* @remarks No-long-jump zone!!!
3482	*/
3483	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3484	{
3485	/*
3486	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3487	*/
3488	uint32_t uIntrState = 0;
3489	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3490	{
3491	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3492	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3493	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3494	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3495	{
3496	if (pMixedCtx->eflags.Bits.u1IF)
3497	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3498	else
3499	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3500	}
3501	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3502	{
3503	/*
3504	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3505	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3506	*/
3507	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3508	}
3509	}
3510
3511	/*
3512	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3513	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3514	* setting this would block host-NMIs and IRET will not clear the blocking.
3515	*
3516	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3517	*/
3518	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3519	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3520	{
3521	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3522	}
3523
3524	return uIntrState;
3525	}
3526
3527
3528	/**
3529	* Loads the guest's interruptibility-state into the guest-state area in the
3530	* VMCS.
3531	*
3532	* @returns VBox status code.
3533	* @param pVCpu The cross context virtual CPU structure.
3534	* @param uIntrState The interruptibility-state to set.
3535	*/
3536	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3537	{
3538	NOREF(pVCpu);
3539	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3540	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3541	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3542	AssertRC(rc);
3543	return rc;
3544	}
3545
3546
3547	/**
3548	* Loads the exception intercepts required for guest execution in the VMCS.
3549	*
3550	* @returns VBox status code.
3551	* @param pVCpu The cross context virtual CPU structure.
3552	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3553	* out-of-sync. Make sure to update the required fields
3554	* before using them.
3555	*/
3556	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3557	{
3558	NOREF(pMixedCtx);
3559	int rc = VINF_SUCCESS;
3560	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3561	{
3562	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3563	if (pVCpu->hm.s.fGIMTrapXcptUD)
3564	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3565	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3566	else
3567	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3568	#endif
3569
3570	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3571	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3572
3573	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3574	AssertRCReturn(rc, rc);
3575
3576	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3577	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3578	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3579	}
3580	return rc;
3581	}
3582
3583
3584	/**
3585	* Loads the guest's RIP into the guest-state area in the VMCS.
3586	*
3587	* @returns VBox status code.
3588	* @param pVCpu The cross context virtual CPU structure.
3589	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590	* out-of-sync. Make sure to update the required fields
3591	* before using them.
3592	*
3593	* @remarks No-long-jump zone!!!
3594	*/
3595	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596	{
3597	int rc = VINF_SUCCESS;
3598	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3599	{
3600	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3601	AssertRCReturn(rc, rc);
3602
3603	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3604	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3605	HMCPU_CF_VALUE(pVCpu)));
3606	}
3607	return rc;
3608	}
3609
3610
3611	/**
3612	* Loads the guest's RSP into the guest-state area in the VMCS.
3613	*
3614	* @returns VBox status code.
3615	* @param pVCpu The cross context virtual CPU structure.
3616	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3617	* out-of-sync. Make sure to update the required fields
3618	* before using them.
3619	*
3620	* @remarks No-long-jump zone!!!
3621	*/
3622	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3623	{
3624	int rc = VINF_SUCCESS;
3625	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3626	{
3627	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3628	AssertRCReturn(rc, rc);
3629
3630	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3631	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3632	}
3633	return rc;
3634	}
3635
3636
3637	/**
3638	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3639	*
3640	* @returns VBox status code.
3641	* @param pVCpu The cross context virtual CPU structure.
3642	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3643	* out-of-sync. Make sure to update the required fields
3644	* before using them.
3645	*
3646	* @remarks No-long-jump zone!!!
3647	*/
3648	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3649	{
3650	int rc = VINF_SUCCESS;
3651	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3652	{
3653	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3654	Let us assert it as such and use 32-bit VMWRITE. */
3655	Assert(!(pMixedCtx->rflags.u64 >> 32));
3656	X86EFLAGS Eflags = pMixedCtx->eflags;
3657	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3658	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3659	* These will never be cleared/set, unless some other part of the VMM
3660	* code is buggy - in which case we're better of finding and fixing
3661	* those bugs than hiding them. */
3662	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3663	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3664	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3665	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3666
3667	/*
3668	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3669	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3670	*/
3671	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3672	{
3673	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3674	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3675	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3676	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3677	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3678	}
3679
3680	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3681	AssertRCReturn(rc, rc);
3682
3683	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3684	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3685	}
3686	return rc;
3687	}
3688
3689
3690	/**
3691	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3692	*
3693	* @returns VBox status code.
3694	* @param pVCpu The cross context virtual CPU structure.
3695	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3696	* out-of-sync. Make sure to update the required fields
3697	* before using them.
3698	*
3699	* @remarks No-long-jump zone!!!
3700	*/
3701	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3702	{
3703	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3704	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3705	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3706	AssertRCReturn(rc, rc);
3707	return rc;
3708	}
3709
3710
3711	/**
3712	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3713	* CR0 is partially shared with the host and we have to consider the FPU bits.
3714	*
3715	* @returns VBox status code.
3716	* @param pVCpu The cross context virtual CPU structure.
3717	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3718	* out-of-sync. Make sure to update the required fields
3719	* before using them.
3720	*
3721	* @remarks No-long-jump zone!!!
3722	*/
3723	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3724	{
3725	/*
3726	* Guest CR0.
3727	* Guest FPU.
3728	*/
3729	int rc = VINF_SUCCESS;
3730	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3731	{
3732	Assert(!(pMixedCtx->cr0 >> 32));
3733	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3734	PVM pVM = pVCpu->CTX_SUFF(pVM);
3735
3736	/* The guest's view (read access) of its CR0 is unblemished. */
3737	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3738	AssertRCReturn(rc, rc);
3739	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3740
3741	/* Setup VT-x's view of the guest CR0. */
3742	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3743	if (pVM->hm.s.fNestedPaging)
3744	{
3745	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3746	{
3747	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3748	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3749	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3750	}
3751	else
3752	{
3753	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3754	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3755	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3756	}
3757
3758	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3759	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3760	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3761
3762	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3763	AssertRCReturn(rc, rc);
3764	}
3765	else
3766	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3767
3768	/*
3769	* Guest FPU bits.
3770	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3771	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3772	*/
3773	u32GuestCR0 \|= X86_CR0_NE;
3774	bool fInterceptNM = false;
3775	if (CPUMIsGuestFPUStateActive(pVCpu))
3776	{
3777	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3778	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3779	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3780	}
3781	else
3782	{
3783	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3784	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3785	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3786	}
3787
3788	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3789	bool fInterceptMF = false;
3790	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3791	fInterceptMF = true;
3792
3793	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3794	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3795	{
3796	Assert(PDMVmmDevHeapIsEnabled(pVM));
3797	Assert(pVM->hm.s.vmx.pRealModeTSS);
3798	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3799	fInterceptNM = true;
3800	fInterceptMF = true;
3801	}
3802	else
3803	{
3804	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3805	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3806	}
3807	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3808
3809	if (fInterceptNM)
3810	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3811	else
3812	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3813
3814	if (fInterceptMF)
3815	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3816	else
3817	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3818
3819	/* Additional intercepts for debugging, define these yourself explicitly. */
3820	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3821	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3822	\| RT_BIT(X86_XCPT_BP)
3823	\| RT_BIT(X86_XCPT_DE)
3824	\| RT_BIT(X86_XCPT_NM)
3825	\| RT_BIT(X86_XCPT_TS)
3826	\| RT_BIT(X86_XCPT_UD)
3827	\| RT_BIT(X86_XCPT_NP)
3828	\| RT_BIT(X86_XCPT_SS)
3829	\| RT_BIT(X86_XCPT_GP)
3830	\| RT_BIT(X86_XCPT_PF)
3831	\| RT_BIT(X86_XCPT_MF)
3832	;
3833	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3834	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3835	#endif
3836
3837	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3838
3839	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3840	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3841	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3842	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3843	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3844	else
3845	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3846
3847	u32GuestCR0 \|= uSetCR0;
3848	u32GuestCR0 &= uZapCR0;
3849	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3850
3851	/* Write VT-x's view of the guest CR0 into the VMCS. */
3852	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3853	AssertRCReturn(rc, rc);
3854	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3855	uZapCR0));
3856
3857	/*
3858	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3859	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3860	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3861	*/
3862	uint32_t u32CR0Mask = 0;
3863	u32CR0Mask = X86_CR0_PE
3864	\| X86_CR0_NE
3865	\| X86_CR0_WP
3866	\| X86_CR0_PG
3867	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3868	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3869	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3870
3871	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3872	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3873	* and @bugref{6944}. */
3874	#if 0
3875	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3876	u32CR0Mask &= ~X86_CR0_PE;
3877	#endif
3878	if (pVM->hm.s.fNestedPaging)
3879	u32CR0Mask &= ~X86_CR0_WP;
3880
3881	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3882	if (fInterceptNM)
3883	{
3884	u32CR0Mask \|= X86_CR0_TS
3885	\| X86_CR0_MP;
3886	}
3887
3888	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3889	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3890	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3891	AssertRCReturn(rc, rc);
3892	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3893
3894	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3895	}
3896	return rc;
3897	}
3898
3899
3900	/**
3901	* Loads the guest control registers (CR3, CR4) into the guest-state area
3902	* in the VMCS.
3903	*
3904	* @returns VBox strict status code.
3905	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3906	* without unrestricted guest access and the VMMDev is not presently
3907	* mapped (e.g. EFI32).
3908	*
3909	* @param pVCpu The cross context virtual CPU structure.
3910	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3911	* out-of-sync. Make sure to update the required fields
3912	* before using them.
3913	*
3914	* @remarks No-long-jump zone!!!
3915	*/
3916	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3917	{
3918	int rc = VINF_SUCCESS;
3919	PVM pVM = pVCpu->CTX_SUFF(pVM);
3920
3921	/*
3922	* Guest CR2.
3923	* It's always loaded in the assembler code. Nothing to do here.
3924	*/
3925
3926	/*
3927	* Guest CR3.
3928	*/
3929	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3930	{
3931	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3932	if (pVM->hm.s.fNestedPaging)
3933	{
3934	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3935
3936	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3937	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3938	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3939	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3940
3941	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3942	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3943	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3944
3945	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3946	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3947	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3948	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3949	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3950	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3951	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3952
3953	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3954	AssertRCReturn(rc, rc);
3955	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3956
3957	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3958	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3959	{
3960	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3961	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3962	{
3963	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3964	AssertRCReturn(rc, rc);
3965	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3966	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3967	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3968	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3969	AssertRCReturn(rc, rc);
3970	}
3971
3972	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3973	have Unrestricted Execution to handle the guest when it's not using paging. */
3974	GCPhysGuestCR3 = pMixedCtx->cr3;
3975	}
3976	else
3977	{
3978	/*
3979	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3980	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3981	* EPT takes care of translating it to host-physical addresses.
3982	*/
3983	RTGCPHYS GCPhys;
3984	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3985
3986	/* We obtain it here every time as the guest could have relocated this PCI region. */
3987	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3988	if (RT_SUCCESS(rc))
3989	{ /* likely */ }
3990	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3991	{
3992	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3993	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3994	}
3995	else
3996	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3997
3998	GCPhysGuestCR3 = GCPhys;
3999	}
4000
4001	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4002	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4003	}
4004	else
4005	{
4006	/* Non-nested paging case, just use the hypervisor's CR3. */
4007	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4008
4009	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4010	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4011	}
4012	AssertRCReturn(rc, rc);
4013
4014	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4015	}
4016
4017	/*
4018	* Guest CR4.
4019	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4020	*/
4021	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4022	{
4023	Assert(!(pMixedCtx->cr4 >> 32));
4024	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4025
4026	/* The guest's view of its CR4 is unblemished. */
4027	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4028	AssertRCReturn(rc, rc);
4029	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4030
4031	/* Setup VT-x's view of the guest CR4. */
4032	/*
4033	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4034	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4035	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4036	*/
4037	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4038	{
4039	Assert(pVM->hm.s.vmx.pRealModeTSS);
4040	Assert(PDMVmmDevHeapIsEnabled(pVM));
4041	u32GuestCR4 &= ~X86_CR4_VME;
4042	}
4043
4044	if (pVM->hm.s.fNestedPaging)
4045	{
4046	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4047	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4048	{
4049	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4050	u32GuestCR4 \|= X86_CR4_PSE;
4051	/* Our identity mapping is a 32-bit page directory. */
4052	u32GuestCR4 &= ~X86_CR4_PAE;
4053	}
4054	/* else use guest CR4.*/
4055	}
4056	else
4057	{
4058	/*
4059	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4060	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4061	*/
4062	switch (pVCpu->hm.s.enmShadowMode)
4063	{
4064	case PGMMODE_REAL: /* Real-mode. */
4065	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4066	case PGMMODE_32_BIT: /* 32-bit paging. */
4067	{
4068	u32GuestCR4 &= ~X86_CR4_PAE;
4069	break;
4070	}
4071
4072	case PGMMODE_PAE: /* PAE paging. */
4073	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4074	{
4075	u32GuestCR4 \|= X86_CR4_PAE;
4076	break;
4077	}
4078
4079	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4080	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4081	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4082	break;
4083	#endif
4084	default:
4085	AssertFailed();
4086	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4087	}
4088	}
4089
4090	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4091	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4092	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4093	u32GuestCR4 \|= uSetCR4;
4094	u32GuestCR4 &= uZapCR4;
4095
4096	/* Write VT-x's view of the guest CR4 into the VMCS. */
4097	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4098	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4099	AssertRCReturn(rc, rc);
4100
4101	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4102	uint32_t u32CR4Mask = X86_CR4_VME
4103	\| X86_CR4_PAE
4104	\| X86_CR4_PGE
4105	\| X86_CR4_PSE
4106	\| X86_CR4_VMXE;
4107	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4108	u32CR4Mask \|= X86_CR4_OSXSAVE;
4109	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4110	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4111	AssertRCReturn(rc, rc);
4112
4113	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4114	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4115
4116	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4117	}
4118	return rc;
4119	}
4120
4121
4122	/**
4123	* Loads the guest debug registers into the guest-state area in the VMCS.
4124	*
4125	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4126	*
4127	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4128	*
4129	* @returns VBox status code.
4130	* @param pVCpu The cross context virtual CPU structure.
4131	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4132	* out-of-sync. Make sure to update the required fields
4133	* before using them.
4134	*
4135	* @remarks No-long-jump zone!!!
4136	*/
4137	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4138	{
4139	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4140	return VINF_SUCCESS;
4141
4142	#ifdef VBOX_STRICT
4143	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4144	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4145	{
4146	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4147	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4148	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4149	}
4150	#endif
4151
4152	int rc;
4153	PVM pVM = pVCpu->CTX_SUFF(pVM);
4154	bool fSteppingDB = false;
4155	bool fInterceptMovDRx = false;
4156	if (pVCpu->hm.s.fSingleInstruction)
4157	{
4158	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4159	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4160	{
4161	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4162	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4163	AssertRCReturn(rc, rc);
4164	Assert(fSteppingDB == false);
4165	}
4166	else
4167	{
4168	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4169	pVCpu->hm.s.fClearTrapFlag = true;
4170	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4171	fSteppingDB = true;
4172	}
4173	}
4174
4175	if ( fSteppingDB
4176	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4177	{
4178	/*
4179	* Use the combined guest and host DRx values found in the hypervisor
4180	* register set because the debugger has breakpoints active or someone
4181	* is single stepping on the host side without a monitor trap flag.
4182	*
4183	* Note! DBGF expects a clean DR6 state before executing guest code.
4184	*/
4185	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4186	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4187	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4188	{
4189	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4190	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4191	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4192	}
4193	else
4194	#endif
4195	if (!CPUMIsHyperDebugStateActive(pVCpu))
4196	{
4197	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4198	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4199	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4200	}
4201
4202	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4203	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4204	AssertRCReturn(rc, rc);
4205
4206	pVCpu->hm.s.fUsingHyperDR7 = true;
4207	fInterceptMovDRx = true;
4208	}
4209	else
4210	{
4211	/*
4212	* If the guest has enabled debug registers, we need to load them prior to
4213	* executing guest code so they'll trigger at the right time.
4214	*/
4215	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4216	{
4217	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4218	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4219	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4220	{
4221	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4222	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4223	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4224	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4225	}
4226	else
4227	#endif
4228	if (!CPUMIsGuestDebugStateActive(pVCpu))
4229	{
4230	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4231	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4232	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4233	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4234	}
4235	Assert(!fInterceptMovDRx);
4236	}
4237	/*
4238	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4239	* must intercept #DB in order to maintain a correct DR6 guest value, and
4240	* because we need to intercept it to prevent nested #DBs from hanging the
4241	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4242	*/
4243	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4244	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4245	&& !CPUMIsGuestDebugStateActive(pVCpu))
4246	#else
4247	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4248	#endif
4249	{
4250	fInterceptMovDRx = true;
4251	}
4252
4253	/* Update guest DR7. */
4254	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4255	AssertRCReturn(rc, rc);
4256
4257	pVCpu->hm.s.fUsingHyperDR7 = false;
4258	}
4259
4260	/*
4261	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4262	*/
4263	if (fInterceptMovDRx)
4264	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4265	else
4266	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4267	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4268	AssertRCReturn(rc, rc);
4269
4270	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4271	return VINF_SUCCESS;
4272	}
4273
4274
4275	#ifdef VBOX_STRICT
4276	/**
4277	* Strict function to validate segment registers.
4278	*
4279	* @remarks ASSUMES CR0 is up to date.
4280	*/
4281	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4282	{
4283	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4284	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4285	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4286	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4287	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4288	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4289	{
4290	/* Protected mode checks */
4291	/* CS */
4292	Assert(pCtx->cs.Attr.n.u1Present);
4293	Assert(!(pCtx->cs.Attr.u & 0xf00));
4294	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4295	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4296	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4297	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4298	\|\| (pCtx->cs.Attr.n.u1Granularity));
4299	/* CS cannot be loaded with NULL in protected mode. */
4300	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4301	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4302	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4303	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4304	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4305	else
4306	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4307	/* SS */
4308	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4309	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4310	if ( !(pCtx->cr0 & X86_CR0_PE)
4311	\|\| pCtx->cs.Attr.n.u4Type == 3)
4312	{
4313	Assert(!pCtx->ss.Attr.n.u2Dpl);
4314	}
4315	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4316	{
4317	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4318	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4319	Assert(pCtx->ss.Attr.n.u1Present);
4320	Assert(!(pCtx->ss.Attr.u & 0xf00));
4321	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4322	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4323	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4324	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4325	\|\| (pCtx->ss.Attr.n.u1Granularity));
4326	}
4327	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4328	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4329	{
4330	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4331	Assert(pCtx->ds.Attr.n.u1Present);
4332	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4333	Assert(!(pCtx->ds.Attr.u & 0xf00));
4334	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4335	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4336	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4337	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4338	\|\| (pCtx->ds.Attr.n.u1Granularity));
4339	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4340	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4341	}
4342	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4343	{
4344	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4345	Assert(pCtx->es.Attr.n.u1Present);
4346	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4347	Assert(!(pCtx->es.Attr.u & 0xf00));
4348	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4349	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4350	\|\| !(pCtx->es.Attr.n.u1Granularity));
4351	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4352	\|\| (pCtx->es.Attr.n.u1Granularity));
4353	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4354	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4355	}
4356	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4357	{
4358	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4359	Assert(pCtx->fs.Attr.n.u1Present);
4360	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4361	Assert(!(pCtx->fs.Attr.u & 0xf00));
4362	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4363	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4364	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4365	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4366	\|\| (pCtx->fs.Attr.n.u1Granularity));
4367	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4368	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4369	}
4370	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4371	{
4372	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4373	Assert(pCtx->gs.Attr.n.u1Present);
4374	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4375	Assert(!(pCtx->gs.Attr.u & 0xf00));
4376	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4377	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4378	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4379	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4380	\|\| (pCtx->gs.Attr.n.u1Granularity));
4381	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4382	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4383	}
4384	/* 64-bit capable CPUs. */
4385	# if HC_ARCH_BITS == 64
4386	Assert(!(pCtx->cs.u64Base >> 32));
4387	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4388	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4389	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4390	# endif
4391	}
4392	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4393	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4394	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4395	{
4396	/* Real and v86 mode checks. */
4397	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4398	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4399	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4400	{
4401	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4402	}
4403	else
4404	{
4405	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4406	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4407	}
4408
4409	/* CS */
4410	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4411	Assert(pCtx->cs.u32Limit == 0xffff);
4412	Assert(u32CSAttr == 0xf3);
4413	/* SS */
4414	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4415	Assert(pCtx->ss.u32Limit == 0xffff);
4416	Assert(u32SSAttr == 0xf3);
4417	/* DS */
4418	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4419	Assert(pCtx->ds.u32Limit == 0xffff);
4420	Assert(u32DSAttr == 0xf3);
4421	/* ES */
4422	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4423	Assert(pCtx->es.u32Limit == 0xffff);
4424	Assert(u32ESAttr == 0xf3);
4425	/* FS */
4426	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4427	Assert(pCtx->fs.u32Limit == 0xffff);
4428	Assert(u32FSAttr == 0xf3);
4429	/* GS */
4430	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4431	Assert(pCtx->gs.u32Limit == 0xffff);
4432	Assert(u32GSAttr == 0xf3);
4433	/* 64-bit capable CPUs. */
4434	# if HC_ARCH_BITS == 64
4435	Assert(!(pCtx->cs.u64Base >> 32));
4436	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4437	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4438	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4439	# endif
4440	}
4441	}
4442	#endif /* VBOX_STRICT */
4443
4444
4445	/**
4446	* Writes a guest segment register into the guest-state area in the VMCS.
4447	*
4448	* @returns VBox status code.
4449	* @param pVCpu The cross context virtual CPU structure.
4450	* @param idxSel Index of the selector in the VMCS.
4451	* @param idxLimit Index of the segment limit in the VMCS.
4452	* @param idxBase Index of the segment base in the VMCS.
4453	* @param idxAccess Index of the access rights of the segment in the VMCS.
4454	* @param pSelReg Pointer to the segment selector.
4455	*
4456	* @remarks No-long-jump zone!!!
4457	*/
4458	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4459	uint32_t idxAccess, PCPUMSELREG pSelReg)
4460	{
4461	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4462	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4463	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4464	AssertRCReturn(rc, rc);
4465
4466	uint32_t u32Access = pSelReg->Attr.u;
4467	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4468	{
4469	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4470	u32Access = 0xf3;
4471	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4472	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4473	}
4474	else
4475	{
4476	/*
4477	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4478	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4479	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4480	* loaded in protected-mode have their attribute as 0.
4481	*/
4482	if (!u32Access)
4483	u32Access = X86DESCATTR_UNUSABLE;
4484	}
4485
4486	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4487	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4488	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4489
4490	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4491	AssertRCReturn(rc, rc);
4492	return rc;
4493	}
4494
4495
4496	/**
4497	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4498	* into the guest-state area in the VMCS.
4499	*
4500	* @returns VBox status code.
4501	* @param pVCpu The cross context virtual CPU structure.
4502	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4503	* out-of-sync. Make sure to update the required fields
4504	* before using them.
4505	*
4506	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4507	* @remarks No-long-jump zone!!!
4508	*/
4509	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4510	{
4511	int rc = VERR_INTERNAL_ERROR_5;
4512	PVM pVM = pVCpu->CTX_SUFF(pVM);
4513
4514	/*
4515	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4516	*/
4517	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4518	{
4519	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4520	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4521	{
4522	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4523	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4524	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4525	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4526	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4527	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4528	}
4529
4530	#ifdef VBOX_WITH_REM
4531	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4532	{
4533	Assert(pVM->hm.s.vmx.pRealModeTSS);
4534	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4535	if ( pVCpu->hm.s.vmx.fWasInRealMode
4536	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4537	{
4538	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4539	in real-mode (e.g. OpenBSD 4.0) */
4540	REMFlushTBs(pVM);
4541	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4542	pVCpu->hm.s.vmx.fWasInRealMode = false;
4543	}
4544	}
4545	#endif
4546	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4547	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4548	AssertRCReturn(rc, rc);
4549	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4550	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4551	AssertRCReturn(rc, rc);
4552	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4553	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4554	AssertRCReturn(rc, rc);
4555	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4556	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4557	AssertRCReturn(rc, rc);
4558	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4559	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4560	AssertRCReturn(rc, rc);
4561	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4562	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4563	AssertRCReturn(rc, rc);
4564
4565	#ifdef VBOX_STRICT
4566	/* Validate. */
4567	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4568	#endif
4569
4570	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4571	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4572	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4573	}
4574
4575	/*
4576	* Guest TR.
4577	*/
4578	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4579	{
4580	/*
4581	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4582	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4583	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4584	*/
4585	uint16_t u16Sel = 0;
4586	uint32_t u32Limit = 0;
4587	uint64_t u64Base = 0;
4588	uint32_t u32AccessRights = 0;
4589
4590	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4591	{
4592	u16Sel = pMixedCtx->tr.Sel;
4593	u32Limit = pMixedCtx->tr.u32Limit;
4594	u64Base = pMixedCtx->tr.u64Base;
4595	u32AccessRights = pMixedCtx->tr.Attr.u;
4596	}
4597	else
4598	{
4599	Assert(pVM->hm.s.vmx.pRealModeTSS);
4600	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4601
4602	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4603	RTGCPHYS GCPhys;
4604	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4605	AssertRCReturn(rc, rc);
4606
4607	X86DESCATTR DescAttr;
4608	DescAttr.u = 0;
4609	DescAttr.n.u1Present = 1;
4610	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4611
4612	u16Sel = 0;
4613	u32Limit = HM_VTX_TSS_SIZE;
4614	u64Base = GCPhys; /* in real-mode phys = virt. */
4615	u32AccessRights = DescAttr.u;
4616	}
4617
4618	/* Validate. */
4619	Assert(!(u16Sel & RT_BIT(2)));
4620	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4621	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4622	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4623	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4624	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4625	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4626	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4627	Assert( (u32Limit & 0xfff) == 0xfff
4628	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4629	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4630	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4631
4632	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4633	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4634	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4635	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4636	AssertRCReturn(rc, rc);
4637
4638	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4639	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4640	}
4641
4642	/*
4643	* Guest GDTR.
4644	*/
4645	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4646	{
4647	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4648	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4649	AssertRCReturn(rc, rc);
4650
4651	/* Validate. */
4652	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4653
4654	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4655	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4656	}
4657
4658	/*
4659	* Guest LDTR.
4660	*/
4661	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4662	{
4663	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4664	uint32_t u32Access = 0;
4665	if (!pMixedCtx->ldtr.Attr.u)
4666	u32Access = X86DESCATTR_UNUSABLE;
4667	else
4668	u32Access = pMixedCtx->ldtr.Attr.u;
4669
4670	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4671	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4672	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4673	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4674	AssertRCReturn(rc, rc);
4675
4676	/* Validate. */
4677	if (!(u32Access & X86DESCATTR_UNUSABLE))
4678	{
4679	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4680	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4681	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4682	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4683	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4684	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4685	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4686	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4687	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4688	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4689	}
4690
4691	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4692	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4693	}
4694
4695	/*
4696	* Guest IDTR.
4697	*/
4698	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4699	{
4700	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4701	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4702	AssertRCReturn(rc, rc);
4703
4704	/* Validate. */
4705	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4706
4707	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4708	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4709	}
4710
4711	return VINF_SUCCESS;
4712	}
4713
4714
4715	/**
4716	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4717	* areas.
4718	*
4719	* These MSRs will automatically be loaded to the host CPU on every successful
4720	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4721	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4722	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4723	*
4724	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4725	*
4726	* @returns VBox status code.
4727	* @param pVCpu The cross context virtual CPU structure.
4728	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4729	* out-of-sync. Make sure to update the required fields
4730	* before using them.
4731	*
4732	* @remarks No-long-jump zone!!!
4733	*/
4734	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4735	{
4736	AssertPtr(pVCpu);
4737	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4738
4739	/*
4740	* MSRs that we use the auto-load/store MSR area in the VMCS.
4741	*/
4742	PVM pVM = pVCpu->CTX_SUFF(pVM);
4743	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4744	{
4745	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4746	#if HC_ARCH_BITS == 32
4747	if (pVM->hm.s.fAllow64BitGuests)
4748	{
4749	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4750	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4751	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4752	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4753	AssertRCReturn(rc, rc);
4754	# ifdef LOG_ENABLED
4755	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4756	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4757	{
4758	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4759	pMsr->u64Value));
4760	}
4761	# endif
4762	}
4763	#endif
4764	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4765	}
4766
4767	/*
4768	* Guest Sysenter MSRs.
4769	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4770	* VM-exits on WRMSRs for these MSRs.
4771	*/
4772	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4773	{
4774	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4775	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4776	}
4777
4778	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4779	{
4780	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4781	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4782	}
4783
4784	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4785	{
4786	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4787	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4788	}
4789
4790	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4791	{
4792	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4793	{
4794	/*
4795	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4796	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4797	*/
4798	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4799	{
4800	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4801	AssertRCReturn(rc,rc);
4802	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4803	}
4804	else
4805	{
4806	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4807	NULL /* pfAddedAndUpdated */);
4808	AssertRCReturn(rc, rc);
4809
4810	/* We need to intercept reads too, see @bugref{7386#c16}. */
4811	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4812	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4813	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4814	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4815	}
4816	}
4817	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4818	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4819	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4820	}
4821
4822	return VINF_SUCCESS;
4823	}
4824
4825
4826	/**
4827	* Loads the guest activity state into the guest-state area in the VMCS.
4828	*
4829	* @returns VBox status code.
4830	* @param pVCpu The cross context virtual CPU structure.
4831	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4832	* out-of-sync. Make sure to update the required fields
4833	* before using them.
4834	*
4835	* @remarks No-long-jump zone!!!
4836	*/
4837	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4838	{
4839	NOREF(pMixedCtx);
4840	/** @todo See if we can make use of other states, e.g.
4841	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4842	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4843	{
4844	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4845	AssertRCReturn(rc, rc);
4846
4847	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4848	}
4849	return VINF_SUCCESS;
4850	}
4851
4852
4853	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4854	/**
4855	* Check if guest state allows safe use of 32-bit switcher again.
4856	*
4857	* Segment bases and protected mode structures must be 32-bit addressable
4858	* because the 32-bit switcher will ignore high dword when writing these VMCS
4859	* fields. See @bugref{8432} for details.
4860	*
4861	* @returns true if safe, false if must continue to use the 64-bit switcher.
4862	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4863	* out-of-sync. Make sure to update the required fields
4864	* before using them.
4865	*
4866	* @remarks No-long-jump zone!!!
4867	*/
4868	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4869	{
4870	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4871	return false;
4872	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4873	return false;
4874	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4875	return false;
4876	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4877	return false;
4878	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4879	return false;
4880	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4881	return false;
4882	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4883	return false;
4884	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4885	return false;
4886	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4887	return false;
4888	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4889	return false;
4890	/* All good, bases are 32-bit. */
4891	return true;
4892	}
4893	#endif
4894
4895
4896	/**
4897	* Sets up the appropriate function to run guest code.
4898	*
4899	* @returns VBox status code.
4900	* @param pVCpu The cross context virtual CPU structure.
4901	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4902	* out-of-sync. Make sure to update the required fields
4903	* before using them.
4904	*
4905	* @remarks No-long-jump zone!!!
4906	*/
4907	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4908	{
4909	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4910	{
4911	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4912	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4913	#endif
4914	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4915	#if HC_ARCH_BITS == 32
4916	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4917	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4918	{
4919	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4920	{
4921	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4922	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4923	\| HM_CHANGED_VMX_ENTRY_CTLS
4924	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4925	}
4926	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4927
4928	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4929	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4930	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4931	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4932	}
4933	#else
4934	/* 64-bit host. */
4935	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4936	#endif
4937	}
4938	else
4939	{
4940	/* Guest is not in long mode, use the 32-bit handler. */
4941	#if HC_ARCH_BITS == 32
4942	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4943	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4944	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4945	{
4946	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4947	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4948	\| HM_CHANGED_VMX_ENTRY_CTLS
4949	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4950	}
4951	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4952	/*
4953	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4954	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4955	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4956	* the much faster 32-bit switcher again.
4957	*/
4958	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4959	{
4960	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4961	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4962	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4963	}
4964	else
4965	{
4966	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4967	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4968	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4969	{
4970	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4971	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4972	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4973	\| HM_CHANGED_VMX_ENTRY_CTLS
4974	\| HM_CHANGED_VMX_EXIT_CTLS
4975	\| HM_CHANGED_HOST_CONTEXT);
4976	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4977	}
4978	}
4979	# else
4980	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4981	# endif
4982	#else
4983	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4984	#endif
4985	}
4986	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4987	return VINF_SUCCESS;
4988	}
4989
4990
4991	/**
4992	* Wrapper for running the guest code in VT-x.
4993	*
4994	* @returns VBox status code, no informational status codes.
4995	* @param pVM The cross context VM structure.
4996	* @param pVCpu The cross context virtual CPU structure.
4997	* @param pCtx Pointer to the guest-CPU context.
4998	*
4999	* @remarks No-long-jump zone!!!
5000	*/
5001	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5002	{
5003	/*
5004	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5005	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5006	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5007	*/
5008	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5009	/** @todo Add stats for resume vs launch. */
5010	#ifdef VBOX_WITH_KERNEL_USING_XMM
5011	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5012	#else
5013	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5014	#endif
5015	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5016	return rc;
5017	}
5018
5019
5020	/**
5021	* Reports world-switch error and dumps some useful debug info.
5022	*
5023	* @param pVM The cross context VM structure.
5024	* @param pVCpu The cross context virtual CPU structure.
5025	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5026	* @param pCtx Pointer to the guest-CPU context.
5027	* @param pVmxTransient Pointer to the VMX transient structure (only
5028	* exitReason updated).
5029	*/
5030	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5031	{
5032	Assert(pVM);
5033	Assert(pVCpu);
5034	Assert(pCtx);
5035	Assert(pVmxTransient);
5036	HMVMX_ASSERT_PREEMPT_SAFE();
5037
5038	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5039	switch (rcVMRun)
5040	{
5041	case VERR_VMX_INVALID_VMXON_PTR:
5042	AssertFailed();
5043	break;
5044	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5045	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5046	{
5047	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5048	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5049	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5050	AssertRC(rc);
5051
5052	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5053	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5054	Cannot do it here as we may have been long preempted. */
5055
5056	#ifdef VBOX_STRICT
5057	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5058	pVmxTransient->uExitReason));
5059	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5060	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5061	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5062	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5063	else
5064	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5065	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5066	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5067
5068	/* VMX control bits. */
5069	uint32_t u32Val;
5070	uint64_t u64Val;
5071	RTHCUINTREG uHCReg;
5072	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5073	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5074	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5075	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5076	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5077	{
5078	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5079	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5080	}
5081	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5082	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5084	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5086	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5088	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5090	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5092	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5094	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5096	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5098	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5100	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5102	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5104	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5106	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5107	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5108	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5109	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5110	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5111	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5112	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5113	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5114	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5115	if (pVM->hm.s.fNestedPaging)
5116	{
5117	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5118	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5119	}
5120
5121	/* Guest bits. */
5122	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5123	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5124	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5125	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5126	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5127	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5128	if (pVM->hm.s.vmx.fVpid)
5129	{
5130	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5131	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5132	}
5133
5134	/* Host bits. */
5135	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5136	Log4(("Host CR0 %#RHr\n", uHCReg));
5137	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5138	Log4(("Host CR3 %#RHr\n", uHCReg));
5139	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5140	Log4(("Host CR4 %#RHr\n", uHCReg));
5141
5142	RTGDTR HostGdtr;
5143	PCX86DESCHC pDesc;
5144	ASMGetGDTR(&HostGdtr);
5145	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5146	Log4(("Host CS %#08x\n", u32Val));
5147	if (u32Val < HostGdtr.cbGdt)
5148	{
5149	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5150	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5151	}
5152
5153	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5154	Log4(("Host DS %#08x\n", u32Val));
5155	if (u32Val < HostGdtr.cbGdt)
5156	{
5157	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5158	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5159	}
5160
5161	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5162	Log4(("Host ES %#08x\n", u32Val));
5163	if (u32Val < HostGdtr.cbGdt)
5164	{
5165	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5166	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5167	}
5168
5169	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5170	Log4(("Host FS %#08x\n", u32Val));
5171	if (u32Val < HostGdtr.cbGdt)
5172	{
5173	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5174	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5175	}
5176
5177	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5178	Log4(("Host GS %#08x\n", u32Val));
5179	if (u32Val < HostGdtr.cbGdt)
5180	{
5181	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5182	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5183	}
5184
5185	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5186	Log4(("Host SS %#08x\n", u32Val));
5187	if (u32Val < HostGdtr.cbGdt)
5188	{
5189	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5190	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5191	}
5192
5193	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5194	Log4(("Host TR %#08x\n", u32Val));
5195	if (u32Val < HostGdtr.cbGdt)
5196	{
5197	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5198	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5199	}
5200
5201	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5202	Log4(("Host TR Base %#RHv\n", uHCReg));
5203	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5204	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5205	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5206	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5207	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5208	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5209	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5210	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5211	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5212	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5213	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5214	Log4(("Host RSP %#RHv\n", uHCReg));
5215	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5216	Log4(("Host RIP %#RHv\n", uHCReg));
5217	# if HC_ARCH_BITS == 64
5218	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5219	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5220	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5221	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5222	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5223	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5224	# endif
5225	#endif /* VBOX_STRICT */
5226	break;
5227	}
5228
5229	default:
5230	/* Impossible */
5231	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5232	break;
5233	}
5234	NOREF(pVM); NOREF(pCtx);
5235	}
5236
5237
5238	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5239	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5240	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5241	#endif
5242	#ifdef VBOX_STRICT
5243	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5244	{
5245	switch (idxField)
5246	{
5247	case VMX_VMCS_GUEST_RIP:
5248	case VMX_VMCS_GUEST_RSP:
5249	case VMX_VMCS_GUEST_SYSENTER_EIP:
5250	case VMX_VMCS_GUEST_SYSENTER_ESP:
5251	case VMX_VMCS_GUEST_GDTR_BASE:
5252	case VMX_VMCS_GUEST_IDTR_BASE:
5253	case VMX_VMCS_GUEST_CS_BASE:
5254	case VMX_VMCS_GUEST_DS_BASE:
5255	case VMX_VMCS_GUEST_ES_BASE:
5256	case VMX_VMCS_GUEST_FS_BASE:
5257	case VMX_VMCS_GUEST_GS_BASE:
5258	case VMX_VMCS_GUEST_SS_BASE:
5259	case VMX_VMCS_GUEST_LDTR_BASE:
5260	case VMX_VMCS_GUEST_TR_BASE:
5261	case VMX_VMCS_GUEST_CR3:
5262	return true;
5263	}
5264	return false;
5265	}
5266
5267	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5268	{
5269	switch (idxField)
5270	{
5271	/* Read-only fields. */
5272	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5273	return true;
5274	}
5275	/* Remaining readable fields should also be writable. */
5276	return hmR0VmxIsValidWriteField(idxField);
5277	}
5278	#endif /* VBOX_STRICT */
5279
5280
5281	/**
5282	* Executes the specified handler in 64-bit mode.
5283	*
5284	* @returns VBox status code (no informational status codes).
5285	* @param pVM The cross context VM structure.
5286	* @param pVCpu The cross context virtual CPU structure.
5287	* @param pCtx Pointer to the guest CPU context.
5288	* @param enmOp The operation to perform.
5289	* @param cParams Number of parameters.
5290	* @param paParam Array of 32-bit parameters.
5291	*/
5292	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5293	uint32_t cParams, uint32_t *paParam)
5294	{
5295	NOREF(pCtx);
5296
5297	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5298	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5299	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5300	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5301
5302	#ifdef VBOX_STRICT
5303	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5304	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5305
5306	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5307	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5308	#endif
5309
5310	/* Disable interrupts. */
5311	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5312
5313	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5314	RTCPUID idHostCpu = RTMpCpuId();
5315	CPUMR0SetLApic(pVCpu, idHostCpu);
5316	#endif
5317
5318	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5319	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5320
5321	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5322	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5323	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5324
5325	/* Leave VMX Root Mode. */
5326	VMXDisable();
5327
5328	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5329
5330	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5331	CPUMSetHyperEIP(pVCpu, enmOp);
5332	for (int i = (int)cParams - 1; i >= 0; i--)
5333	CPUMPushHyper(pVCpu, paParam[i]);
5334
5335	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5336
5337	/* Call the switcher. */
5338	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5339	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5340
5341	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5342	/* Make sure the VMX instructions don't cause #UD faults. */
5343	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5344
5345	/* Re-enter VMX Root Mode */
5346	int rc2 = VMXEnable(HCPhysCpuPage);
5347	if (RT_FAILURE(rc2))
5348	{
5349	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5350	ASMSetFlags(fOldEFlags);
5351	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5352	return rc2;
5353	}
5354
5355	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5356	AssertRC(rc2);
5357	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5358	Assert(!(ASMGetFlags() & X86_EFL_IF));
5359	ASMSetFlags(fOldEFlags);
5360	return rc;
5361	}
5362
5363
5364	/**
5365	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5366	* supporting 64-bit guests.
5367	*
5368	* @returns VBox status code.
5369	* @param fResume Whether to VMLAUNCH or VMRESUME.
5370	* @param pCtx Pointer to the guest-CPU context.
5371	* @param pCache Pointer to the VMCS cache.
5372	* @param pVM The cross context VM structure.
5373	* @param pVCpu The cross context virtual CPU structure.
5374	*/
5375	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5376	{
5377	NOREF(fResume);
5378
5379	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5380	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5381
5382	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5383	pCache->uPos = 1;
5384	pCache->interPD = PGMGetInterPaeCR3(pVM);
5385	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5386	#endif
5387
5388	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5389	pCache->TestIn.HCPhysCpuPage = 0;
5390	pCache->TestIn.HCPhysVmcs = 0;
5391	pCache->TestIn.pCache = 0;
5392	pCache->TestOut.HCPhysVmcs = 0;
5393	pCache->TestOut.pCache = 0;
5394	pCache->TestOut.pCtx = 0;
5395	pCache->TestOut.eflags = 0;
5396	#else
5397	NOREF(pCache);
5398	#endif
5399
5400	uint32_t aParam[10];
5401	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5402	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5403	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5404	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5405	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5406	aParam[5] = 0;
5407	aParam[6] = VM_RC_ADDR(pVM, pVM);
5408	aParam[7] = 0;
5409	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5410	aParam[9] = 0;
5411
5412	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5413	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5414	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5415	#endif
5416	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5417
5418	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5419	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5420	Assert(pCtx->dr[4] == 10);
5421	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5422	#endif
5423
5424	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5425	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5426	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5427	pVCpu->hm.s.vmx.HCPhysVmcs));
5428	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5429	pCache->TestOut.HCPhysVmcs));
5430	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5431	pCache->TestOut.pCache));
5432	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5433	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5434	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5435	pCache->TestOut.pCtx));
5436	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5437	#endif
5438	return rc;
5439	}
5440
5441
5442	/**
5443	* Initialize the VMCS-Read cache.
5444	*
5445	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5446	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5447	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5448	* (those that have a 32-bit FULL & HIGH part).
5449	*
5450	* @returns VBox status code.
5451	* @param pVM The cross context VM structure.
5452	* @param pVCpu The cross context virtual CPU structure.
5453	*/
5454	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5455	{
5456	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5457	{ \
5458	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5459	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5460	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5461	++cReadFields; \
5462	}
5463
5464	AssertPtr(pVM);
5465	AssertPtr(pVCpu);
5466	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5467	uint32_t cReadFields = 0;
5468
5469	/*
5470	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5471	* and serve to indicate exceptions to the rules.
5472	*/
5473
5474	/* Guest-natural selector base fields. */
5475	#if 0
5476	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5477	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5478	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5479	#endif
5480	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5481	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5482	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5483	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5484	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5485	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5486	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5487	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5488	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5489	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5491	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5492	#if 0
5493	/* Unused natural width guest-state fields. */
5494	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5495	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5496	#endif
5497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5498	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5499
5500	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5501	#if 0
5502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5511	#endif
5512
5513	/* Natural width guest-state fields. */
5514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5515	#if 0
5516	/* Currently unused field. */
5517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5518	#endif
5519
5520	if (pVM->hm.s.fNestedPaging)
5521	{
5522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5523	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5524	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5525	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5526	}
5527	else
5528	{
5529	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5530	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5531	}
5532
5533	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5534	return VINF_SUCCESS;
5535	}
5536
5537
5538	/**
5539	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5540	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5541	* darwin, running 64-bit guests).
5542	*
5543	* @returns VBox status code.
5544	* @param pVCpu The cross context virtual CPU structure.
5545	* @param idxField The VMCS field encoding.
5546	* @param u64Val 16, 32 or 64-bit value.
5547	*/
5548	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5549	{
5550	int rc;
5551	switch (idxField)
5552	{
5553	/*
5554	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5555	*/
5556	/* 64-bit Control fields. */
5557	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5558	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5559	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5560	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5561	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5562	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5563	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5564	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5565	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5566	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5567	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5568	case VMX_VMCS64_CTRL_EPTP_FULL:
5569	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5570	/* 64-bit Guest-state fields. */
5571	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5572	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5573	case VMX_VMCS64_GUEST_PAT_FULL:
5574	case VMX_VMCS64_GUEST_EFER_FULL:
5575	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5576	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5577	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5578	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5579	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5580	/* 64-bit Host-state fields. */
5581	case VMX_VMCS64_HOST_PAT_FULL:
5582	case VMX_VMCS64_HOST_EFER_FULL:
5583	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5584	{
5585	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5586	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5587	break;
5588	}
5589
5590	/*
5591	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5592	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5593	*/
5594	/* Natural-width Guest-state fields. */
5595	case VMX_VMCS_GUEST_CR3:
5596	case VMX_VMCS_GUEST_ES_BASE:
5597	case VMX_VMCS_GUEST_CS_BASE:
5598	case VMX_VMCS_GUEST_SS_BASE:
5599	case VMX_VMCS_GUEST_DS_BASE:
5600	case VMX_VMCS_GUEST_FS_BASE:
5601	case VMX_VMCS_GUEST_GS_BASE:
5602	case VMX_VMCS_GUEST_LDTR_BASE:
5603	case VMX_VMCS_GUEST_TR_BASE:
5604	case VMX_VMCS_GUEST_GDTR_BASE:
5605	case VMX_VMCS_GUEST_IDTR_BASE:
5606	case VMX_VMCS_GUEST_RSP:
5607	case VMX_VMCS_GUEST_RIP:
5608	case VMX_VMCS_GUEST_SYSENTER_ESP:
5609	case VMX_VMCS_GUEST_SYSENTER_EIP:
5610	{
5611	if (!(RT_HI_U32(u64Val)))
5612	{
5613	/* If this field is 64-bit, VT-x will zero out the top bits. */
5614	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5615	}
5616	else
5617	{
5618	/* Assert that only the 32->64 switcher case should ever come here. */
5619	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5620	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5621	}
5622	break;
5623	}
5624
5625	default:
5626	{
5627	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5628	rc = VERR_INVALID_PARAMETER;
5629	break;
5630	}
5631	}
5632	AssertRCReturn(rc, rc);
5633	return rc;
5634	}
5635
5636
5637	/**
5638	* Queue up a VMWRITE by using the VMCS write cache.
5639	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5640	*
5641	* @param pVCpu The cross context virtual CPU structure.
5642	* @param idxField The VMCS field encoding.
5643	* @param u64Val 16, 32 or 64-bit value.
5644	*/
5645	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5646	{
5647	AssertPtr(pVCpu);
5648	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5649
5650	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5651	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5652
5653	/* Make sure there are no duplicates. */
5654	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5655	{
5656	if (pCache->Write.aField[i] == idxField)
5657	{
5658	pCache->Write.aFieldVal[i] = u64Val;
5659	return VINF_SUCCESS;
5660	}
5661	}
5662
5663	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5664	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5665	pCache->Write.cValidEntries++;
5666	return VINF_SUCCESS;
5667	}
5668	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5669
5670
5671	/**
5672	* Sets up the usage of TSC-offsetting and updates the VMCS.
5673	*
5674	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5675	* VMX preemption timer.
5676	*
5677	* @returns VBox status code.
5678	* @param pVM The cross context VM structure.
5679	* @param pVCpu The cross context virtual CPU structure.
5680	*
5681	* @remarks No-long-jump zone!!!
5682	*/
5683	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5684	{
5685	int rc;
5686	bool fOffsettedTsc;
5687	bool fParavirtTsc;
5688	if (pVM->hm.s.vmx.fUsePreemptTimer)
5689	{
5690	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5691	&fOffsettedTsc, &fParavirtTsc);
5692
5693	/* Make sure the returned values have sane upper and lower boundaries. */
5694	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5695	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5696	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5697	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5698
5699	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5700	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5701	}
5702	else
5703	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5704
5705	/** @todo later optimize this to be done elsewhere and not before every
5706	* VM-entry. */
5707	if (fParavirtTsc)
5708	{
5709	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5710	information before every VM-entry, hence disable it for performance sake. */
5711	#if 0
5712	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5713	AssertRC(rc);
5714	#endif
5715	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5716	}
5717
5718	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5719	{
5720	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5721	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5722
5723	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5724	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5725	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5726	}
5727	else
5728	{
5729	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5730	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5731	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5732	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5733	}
5734	}
5735
5736
5737	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5738	/**
5739	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5740	* VM-exit interruption info type.
5741	*
5742	* @returns The IEM exception flags.
5743	* @param uVector The event vector.
5744	* @param uVmxVectorType The VMX event type.
5745	*
5746	* @remarks This function currently only constructs flags required for
5747	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5748	* and CR2 aspects of an exception are not included).
5749	*/
5750	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5751	{
5752	uint32_t fIemXcptFlags;
5753	switch (uVmxVectorType)
5754	{
5755	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5756	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5757	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5758	break;
5759
5760	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5761	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5762	break;
5763
5764	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5765	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5766	break;
5767
5768	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5769	{
5770	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5771	if (uVector == X86_XCPT_BP)
5772	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5773	else if (uVector == X86_XCPT_OF)
5774	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5775	else
5776	{
5777	fIemXcptFlags = 0;
5778	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5779	}
5780	break;
5781	}
5782
5783	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5784	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5785	break;
5786
5787	default:
5788	fIemXcptFlags = 0;
5789	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5790	break;
5791	}
5792	return fIemXcptFlags;
5793	}
5794
5795	#else
5796	/**
5797	* Determines if an exception is a contributory exception.
5798	*
5799	* Contributory exceptions are ones which can cause double-faults unless the
5800	* original exception was a benign exception. Page-fault is intentionally not
5801	* included here as it's a conditional contributory exception.
5802	*
5803	* @returns true if the exception is contributory, false otherwise.
5804	* @param uVector The exception vector.
5805	*/
5806	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5807	{
5808	switch (uVector)
5809	{
5810	case X86_XCPT_GP:
5811	case X86_XCPT_SS:
5812	case X86_XCPT_NP:
5813	case X86_XCPT_TS:
5814	case X86_XCPT_DE:
5815	return true;
5816	default:
5817	break;
5818	}
5819	return false;
5820	}
5821	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5822
5823
5824	/**
5825	* Sets an event as a pending event to be injected into the guest.
5826	*
5827	* @param pVCpu The cross context virtual CPU structure.
5828	* @param u32IntInfo The VM-entry interruption-information field.
5829	* @param cbInstr The VM-entry instruction length in bytes (for software
5830	* interrupts, exceptions and privileged software
5831	* exceptions).
5832	* @param u32ErrCode The VM-entry exception error code.
5833	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5834	* page-fault.
5835	*
5836	* @remarks Statistics counter assumes this is a guest event being injected or
5837	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5838	* always incremented.
5839	*/
5840	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5841	RTGCUINTPTR GCPtrFaultAddress)
5842	{
5843	Assert(!pVCpu->hm.s.Event.fPending);
5844	pVCpu->hm.s.Event.fPending = true;
5845	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5846	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5847	pVCpu->hm.s.Event.cbInstr = cbInstr;
5848	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5849	}
5850
5851
5852	/**
5853	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5854	*
5855	* @param pVCpu The cross context virtual CPU structure.
5856	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5857	* out-of-sync. Make sure to update the required fields
5858	* before using them.
5859	*/
5860	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5861	{
5862	NOREF(pMixedCtx);
5863	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5864	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5865	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5866	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5867	}
5868
5869
5870	/**
5871	* Handle a condition that occurred while delivering an event through the guest
5872	* IDT.
5873	*
5874	* @returns Strict VBox status code (i.e. informational status codes too).
5875	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5876	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5877	* to continue execution of the guest which will delivery the \#DF.
5878	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5879	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5880	*
5881	* @param pVCpu The cross context virtual CPU structure.
5882	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5883	* out-of-sync. Make sure to update the required fields
5884	* before using them.
5885	* @param pVmxTransient Pointer to the VMX transient structure.
5886	*
5887	* @remarks No-long-jump zone!!!
5888	*/
5889	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5890	{
5891	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5892
5893	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5894	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5895
5896	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5897	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5898	{
5899	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5900	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5901	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5902	/*
5903	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5904	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5905	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5906	* exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5907	* currently has no way of storing that these exceptions were caused by these instructions
5908	* (ICEBP's #DB poses the problem).
5909	*/
5910	IEMXCPTRAISE enmRaise;
5911	IEMXCPTRAISEINFO fRaiseInfo;
5912	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5913	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5914	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5915	{
5916	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5917	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5918	}
5919	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5920	{
5921	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5922	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5923	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5924	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5925	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5926	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5927	uExitVectorType), VERR_VMX_IPE_5);
5928	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5929
5930	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5931	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5932	{
5933	pVmxTransient->fVectoringPF = true;
5934	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5935	}
5936	}
5937	else
5938	{
5939	/*
5940	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5941	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5942	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5943	*/
5944	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5945	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5946	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5947	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5948	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5949	}
5950
5951	/*
5952	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5953	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5954	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5955	* subsequent VM-entry would fail.
5956	*
5957	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5958	*/
5959	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5960	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5961	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5962	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5963	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5964	{
5965	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5966	}
5967
5968	switch (enmRaise)
5969	{
5970	case IEMXCPTRAISE_CURRENT_XCPT:
5971	{
5972	Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
5973	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5974	Assert(rcStrict == VINF_SUCCESS);
5975	break;
5976	}
5977
5978	case IEMXCPTRAISE_PREV_EVENT:
5979	{
5980	uint32_t u32ErrCode;
5981	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5982	{
5983	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5984	AssertRCReturn(rc2, rc2);
5985	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5986	}
5987	else
5988	u32ErrCode = 0;
5989
5990	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5991	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5992	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5993	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5994
5995	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
5996	pVCpu->hm.s.Event.u32ErrCode));
5997	Assert(rcStrict == VINF_SUCCESS);
5998	break;
5999	}
6000
6001	case IEMXCPTRAISE_REEXEC_INSTR:
6002	Assert(rcStrict == VINF_SUCCESS);
6003	break;
6004
6005	case IEMXCPTRAISE_DOUBLE_FAULT:
6006	{
6007	/*
6008	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6009	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6010	*/
6011	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6012	{
6013	pVmxTransient->fVectoringDoublePF = true;
6014	Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6015	pMixedCtx->cr2));
6016	rcStrict = VINF_SUCCESS;
6017	}
6018	else
6019	{
6020	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6021	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6022	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6023	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6024	rcStrict = VINF_HM_DOUBLE_FAULT;
6025	}
6026	break;
6027	}
6028
6029	case IEMXCPTRAISE_TRIPLE_FAULT:
6030	{
6031	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6032	uExitVector));
6033	rcStrict = VINF_EM_RESET;
6034	break;
6035	}
6036
6037	case IEMXCPTRAISE_CPU_HANG:
6038	{
6039	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6040	rcStrict = VERR_EM_GUEST_CPU_HANG;
6041	break;
6042	}
6043
6044	default:
6045	{
6046	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6047	rcStrict = VERR_VMX_IPE_2;
6048	break;
6049	}
6050	}
6051	#else
6052	typedef enum
6053	{
6054	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6055	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6056	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6057	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6058	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6059	} VMXREFLECTXCPT;
6060
6061	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6062	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6063	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6064	{
6065	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6066	{
6067	enmReflect = VMXREFLECTXCPT_XCPT;
6068	#ifdef VBOX_STRICT
6069	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6070	&& uExitVector == X86_XCPT_PF)
6071	{
6072	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6073	}
6074	#endif
6075	if ( uExitVector == X86_XCPT_PF
6076	&& uIdtVector == X86_XCPT_PF)
6077	{
6078	pVmxTransient->fVectoringDoublePF = true;
6079	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6080	}
6081	else if ( uExitVector == X86_XCPT_AC
6082	&& uIdtVector == X86_XCPT_AC)
6083	{
6084	enmReflect = VMXREFLECTXCPT_HANG;
6085	Log4(("IDT: Nested #AC - Bad guest\n"));
6086	}
6087	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6088	&& hmR0VmxIsContributoryXcpt(uExitVector)
6089	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6090	\|\| uIdtVector == X86_XCPT_PF))
6091	{
6092	enmReflect = VMXREFLECTXCPT_DF;
6093	}
6094	else if (uIdtVector == X86_XCPT_DF)
6095	enmReflect = VMXREFLECTXCPT_TF;
6096	}
6097	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6098	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6099	{
6100	/*
6101	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6102	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6103	*/
6104	enmReflect = VMXREFLECTXCPT_XCPT;
6105
6106	if (uExitVector == X86_XCPT_PF)
6107	{
6108	pVmxTransient->fVectoringPF = true;
6109	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6110	}
6111	}
6112	}
6113	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6114	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6115	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6116	{
6117	/*
6118	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6119	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6120	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6121	*/
6122	enmReflect = VMXREFLECTXCPT_XCPT;
6123	}
6124
6125	/*
6126	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6127	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6128	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6129	*
6130	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6131	*/
6132	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6133	&& enmReflect == VMXREFLECTXCPT_XCPT
6134	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6135	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6136	{
6137	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6138	}
6139
6140	switch (enmReflect)
6141	{
6142	case VMXREFLECTXCPT_XCPT:
6143	{
6144	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6145	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6146	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6147
6148	uint32_t u32ErrCode = 0;
6149	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6150	{
6151	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6152	AssertRCReturn(rc2, rc2);
6153	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6154	}
6155
6156	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6157	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6158	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6159	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6160	rcStrict = VINF_SUCCESS;
6161	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6162	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6163
6164	break;
6165	}
6166
6167	case VMXREFLECTXCPT_DF:
6168	{
6169	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6170	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6171	rcStrict = VINF_HM_DOUBLE_FAULT;
6172	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6173	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6174
6175	break;
6176	}
6177
6178	case VMXREFLECTXCPT_TF:
6179	{
6180	rcStrict = VINF_EM_RESET;
6181	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6182	uExitVector));
6183	break;
6184	}
6185
6186	case VMXREFLECTXCPT_HANG:
6187	{
6188	rcStrict = VERR_EM_GUEST_CPU_HANG;
6189	break;
6190	}
6191
6192	default:
6193	Assert(rcStrict == VINF_SUCCESS);
6194	break;
6195	}
6196	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6197	}
6198	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6199	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6200	&& uExitVector != X86_XCPT_DF
6201	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6202	{
6203	/*
6204	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6205	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6206	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6207	*/
6208	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6209	{
6210	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6211	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6212	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6213	}
6214	}
6215
6216	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6217	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6218	return rcStrict;
6219	}
6220
6221
6222	/**
6223	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6224	*
6225	* @returns VBox status code.
6226	* @param pVCpu The cross context virtual CPU structure.
6227	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6228	* out-of-sync. Make sure to update the required fields
6229	* before using them.
6230	*
6231	* @remarks No-long-jump zone!!!
6232	*/
6233	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6234	{
6235	NOREF(pMixedCtx);
6236
6237	/*
6238	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6239	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6240	*/
6241	VMMRZCallRing3Disable(pVCpu);
6242	HM_DISABLE_PREEMPT();
6243
6244	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6245	{
6246	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6247	* and 'dbgc-init' containing:
6248	* sxe "xcpt_de"
6249	* sxe "xcpt_bp"
6250	* sxi "xcpt_gp"
6251	* sxi "xcpt_ss"
6252	* sxi "xcpt_np"
6253	*/
6254	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6255	#endif
6256	uint32_t uVal = 0;
6257	uint32_t uShadow = 0;
6258	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6259	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6260	AssertRCReturn(rc, rc);
6261
6262	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6263	CPUMSetGuestCR0(pVCpu, uVal);
6264	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6265	}
6266
6267	HM_RESTORE_PREEMPT();
6268	VMMRZCallRing3Enable(pVCpu);
6269	return VINF_SUCCESS;
6270	}
6271
6272
6273	/**
6274	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6275	*
6276	* @returns VBox status code.
6277	* @param pVCpu The cross context virtual CPU structure.
6278	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6279	* out-of-sync. Make sure to update the required fields
6280	* before using them.
6281	*
6282	* @remarks No-long-jump zone!!!
6283	*/
6284	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6285	{
6286	NOREF(pMixedCtx);
6287
6288	int rc = VINF_SUCCESS;
6289	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6290	{
6291	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6292	uint32_t uVal = 0;
6293	uint32_t uShadow = 0;
6294	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6295	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6296	AssertRCReturn(rc, rc);
6297
6298	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6299	CPUMSetGuestCR4(pVCpu, uVal);
6300	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6301	}
6302	return rc;
6303	}
6304
6305
6306	/**
6307	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6308	*
6309	* @returns VBox status code.
6310	* @param pVCpu The cross context virtual CPU structure.
6311	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6312	* out-of-sync. Make sure to update the required fields
6313	* before using them.
6314	*
6315	* @remarks No-long-jump zone!!!
6316	*/
6317	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6318	{
6319	int rc = VINF_SUCCESS;
6320	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6321	{
6322	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6323	uint64_t u64Val = 0;
6324	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6325	AssertRCReturn(rc, rc);
6326
6327	pMixedCtx->rip = u64Val;
6328	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6329	}
6330	return rc;
6331	}
6332
6333
6334	/**
6335	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6336	*
6337	* @returns VBox status code.
6338	* @param pVCpu The cross context virtual CPU structure.
6339	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6340	* out-of-sync. Make sure to update the required fields
6341	* before using them.
6342	*
6343	* @remarks No-long-jump zone!!!
6344	*/
6345	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6346	{
6347	int rc = VINF_SUCCESS;
6348	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6349	{
6350	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6351	uint64_t u64Val = 0;
6352	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6353	AssertRCReturn(rc, rc);
6354
6355	pMixedCtx->rsp = u64Val;
6356	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6357	}
6358	return rc;
6359	}
6360
6361
6362	/**
6363	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6364	*
6365	* @returns VBox status code.
6366	* @param pVCpu The cross context virtual CPU structure.
6367	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6368	* out-of-sync. Make sure to update the required fields
6369	* before using them.
6370	*
6371	* @remarks No-long-jump zone!!!
6372	*/
6373	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6374	{
6375	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6376	{
6377	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6378	uint32_t uVal = 0;
6379	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6380	AssertRCReturn(rc, rc);
6381
6382	pMixedCtx->eflags.u32 = uVal;
6383	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6384	{
6385	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6386	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6387
6388	pMixedCtx->eflags.Bits.u1VM = 0;
6389	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6390	}
6391
6392	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6393	}
6394	return VINF_SUCCESS;
6395	}
6396
6397
6398	/**
6399	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6400	* guest-CPU context.
6401	*/
6402	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6403	{
6404	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6405	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6406	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6407	return rc;
6408	}
6409
6410
6411	/**
6412	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6413	* from the guest-state area in the VMCS.
6414	*
6415	* @param pVCpu The cross context virtual CPU structure.
6416	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6417	* out-of-sync. Make sure to update the required fields
6418	* before using them.
6419	*
6420	* @remarks No-long-jump zone!!!
6421	*/
6422	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6423	{
6424	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6425	{
6426	uint32_t uIntrState = 0;
6427	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6428	AssertRC(rc);
6429
6430	if (!uIntrState)
6431	{
6432	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6433	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6434
6435	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6436	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6437	}
6438	else
6439	{
6440	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6441	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6442	{
6443	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6444	AssertRC(rc);
6445	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6446	AssertRC(rc);
6447
6448	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6449	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6450	}
6451	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6452	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6453
6454	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6455	{
6456	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6457	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6458	}
6459	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6460	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6461	}
6462
6463	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6464	}
6465	}
6466
6467
6468	/**
6469	* Saves the guest's activity state.
6470	*
6471	* @returns VBox status code.
6472	* @param pVCpu The cross context virtual CPU structure.
6473	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6474	* out-of-sync. Make sure to update the required fields
6475	* before using them.
6476	*
6477	* @remarks No-long-jump zone!!!
6478	*/
6479	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6480	{
6481	NOREF(pMixedCtx);
6482	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6483	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6484	return VINF_SUCCESS;
6485	}
6486
6487
6488	/**
6489	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6490	* the current VMCS into the guest-CPU context.
6491	*
6492	* @returns VBox status code.
6493	* @param pVCpu The cross context virtual CPU structure.
6494	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6495	* out-of-sync. Make sure to update the required fields
6496	* before using them.
6497	*
6498	* @remarks No-long-jump zone!!!
6499	*/
6500	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6501	{
6502	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6503	{
6504	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6505	uint32_t u32Val = 0;
6506	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6507	pMixedCtx->SysEnter.cs = u32Val;
6508	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6509	}
6510
6511	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6512	{
6513	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6514	uint64_t u64Val = 0;
6515	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6516	pMixedCtx->SysEnter.eip = u64Val;
6517	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6518	}
6519	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6520	{
6521	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6522	uint64_t u64Val = 0;
6523	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6524	pMixedCtx->SysEnter.esp = u64Val;
6525	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6526	}
6527	return VINF_SUCCESS;
6528	}
6529
6530
6531	/**
6532	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6533	* the CPU back into the guest-CPU context.
6534	*
6535	* @returns VBox status code.
6536	* @param pVCpu The cross context virtual CPU structure.
6537	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6538	* out-of-sync. Make sure to update the required fields
6539	* before using them.
6540	*
6541	* @remarks No-long-jump zone!!!
6542	*/
6543	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6544	{
6545	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6546	VMMRZCallRing3Disable(pVCpu);
6547	HM_DISABLE_PREEMPT();
6548
6549	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6550	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6551	{
6552	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6553	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6554	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6555	}
6556
6557	HM_RESTORE_PREEMPT();
6558	VMMRZCallRing3Enable(pVCpu);
6559
6560	return VINF_SUCCESS;
6561	}
6562
6563
6564	/**
6565	* Saves the auto load/store'd guest MSRs from the current VMCS into
6566	* the guest-CPU context.
6567	*
6568	* @returns VBox status code.
6569	* @param pVCpu The cross context virtual CPU structure.
6570	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6571	* out-of-sync. Make sure to update the required fields
6572	* before using them.
6573	*
6574	* @remarks No-long-jump zone!!!
6575	*/
6576	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6577	{
6578	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6579	return VINF_SUCCESS;
6580
6581	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6582	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6583	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6584	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6585	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6586	{
6587	switch (pMsr->u32Msr)
6588	{
6589	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6590	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6591	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6592	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6593	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6594	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6595	break;
6596
6597	default:
6598	{
6599	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6600	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6601	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6602	}
6603	}
6604	}
6605
6606	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6607	return VINF_SUCCESS;
6608	}
6609
6610
6611	/**
6612	* Saves the guest control registers from the current VMCS into the guest-CPU
6613	* context.
6614	*
6615	* @returns VBox status code.
6616	* @param pVCpu The cross context virtual CPU structure.
6617	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6618	* out-of-sync. Make sure to update the required fields
6619	* before using them.
6620	*
6621	* @remarks No-long-jump zone!!!
6622	*/
6623	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6624	{
6625	/* Guest CR0. Guest FPU. */
6626	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6627	AssertRCReturn(rc, rc);
6628
6629	/* Guest CR4. */
6630	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6631	AssertRCReturn(rc, rc);
6632
6633	/* Guest CR2 - updated always during the world-switch or in #PF. */
6634	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6635	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6636	{
6637	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6638	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6639	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6640
6641	PVM pVM = pVCpu->CTX_SUFF(pVM);
6642	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6643	\|\| ( pVM->hm.s.fNestedPaging
6644	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6645	{
6646	uint64_t u64Val = 0;
6647	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6648	if (pMixedCtx->cr3 != u64Val)
6649	{
6650	CPUMSetGuestCR3(pVCpu, u64Val);
6651	if (VMMRZCallRing3IsEnabled(pVCpu))
6652	{
6653	PGMUpdateCR3(pVCpu, u64Val);
6654	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6655	}
6656	else
6657	{
6658	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6659	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6660	}
6661	}
6662
6663	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6664	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6665	{
6666	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6667	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6668	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6669	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6670	AssertRCReturn(rc, rc);
6671
6672	if (VMMRZCallRing3IsEnabled(pVCpu))
6673	{
6674	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6675	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6676	}
6677	else
6678	{
6679	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6680	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6681	}
6682	}
6683	}
6684
6685	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6686	}
6687
6688	/*
6689	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6690	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6691	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6692	*
6693	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6694	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6695	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6696	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6697	*
6698	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6699	*/
6700	if (VMMRZCallRing3IsEnabled(pVCpu))
6701	{
6702	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6703	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6704
6705	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6706	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6707
6708	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6709	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6710	}
6711
6712	return rc;
6713	}
6714
6715
6716	/**
6717	* Reads a guest segment register from the current VMCS into the guest-CPU
6718	* context.
6719	*
6720	* @returns VBox status code.
6721	* @param pVCpu The cross context virtual CPU structure.
6722	* @param idxSel Index of the selector in the VMCS.
6723	* @param idxLimit Index of the segment limit in the VMCS.
6724	* @param idxBase Index of the segment base in the VMCS.
6725	* @param idxAccess Index of the access rights of the segment in the VMCS.
6726	* @param pSelReg Pointer to the segment selector.
6727	*
6728	* @remarks No-long-jump zone!!!
6729	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6730	* macro as that takes care of whether to read from the VMCS cache or
6731	* not.
6732	*/
6733	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6734	PCPUMSELREG pSelReg)
6735	{
6736	NOREF(pVCpu);
6737
6738	uint32_t u32Val = 0;
6739	int rc = VMXReadVmcs32(idxSel, &u32Val);
6740	AssertRCReturn(rc, rc);
6741	pSelReg->Sel = (uint16_t)u32Val;
6742	pSelReg->ValidSel = (uint16_t)u32Val;
6743	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6744
6745	rc = VMXReadVmcs32(idxLimit, &u32Val);
6746	AssertRCReturn(rc, rc);
6747	pSelReg->u32Limit = u32Val;
6748
6749	uint64_t u64Val = 0;
6750	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6751	AssertRCReturn(rc, rc);
6752	pSelReg->u64Base = u64Val;
6753
6754	rc = VMXReadVmcs32(idxAccess, &u32Val);
6755	AssertRCReturn(rc, rc);
6756	pSelReg->Attr.u = u32Val;
6757
6758	/*
6759	* If VT-x marks the segment as unusable, most other bits remain undefined:
6760	* - For CS the L, D and G bits have meaning.
6761	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6762	* - For the remaining data segments no bits are defined.
6763	*
6764	* The present bit and the unusable bit has been observed to be set at the
6765	* same time (the selector was supposed to be invalid as we started executing
6766	* a V8086 interrupt in ring-0).
6767	*
6768	* What should be important for the rest of the VBox code, is that the P bit is
6769	* cleared. Some of the other VBox code recognizes the unusable bit, but
6770	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6771	* safe side here, we'll strip off P and other bits we don't care about. If
6772	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6773	*
6774	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6775	*/
6776	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6777	{
6778	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6779
6780	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6781	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6782	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6783
6784	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6785	#ifdef DEBUG_bird
6786	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6787	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6788	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6789	#endif
6790	}
6791	return VINF_SUCCESS;
6792	}
6793
6794
6795	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6796	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6797	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6798	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6799	#else
6800	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6801	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6802	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6803	#endif
6804
6805
6806	/**
6807	* Saves the guest segment registers from the current VMCS into the guest-CPU
6808	* context.
6809	*
6810	* @returns VBox status code.
6811	* @param pVCpu The cross context virtual CPU structure.
6812	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6813	* out-of-sync. Make sure to update the required fields
6814	* before using them.
6815	*
6816	* @remarks No-long-jump zone!!!
6817	*/
6818	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6819	{
6820	/* Guest segment registers. */
6821	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6822	{
6823	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6824	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6825	AssertRCReturn(rc, rc);
6826
6827	rc = VMXLOCAL_READ_SEG(CS, cs);
6828	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6829	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6830	rc \|= VMXLOCAL_READ_SEG(ES, es);
6831	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6832	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6833	AssertRCReturn(rc, rc);
6834
6835	/* Restore segment attributes for real-on-v86 mode hack. */
6836	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6837	{
6838	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6839	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6840	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6841	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6842	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6843	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6844	}
6845	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6846	}
6847
6848	return VINF_SUCCESS;
6849	}
6850
6851
6852	/**
6853	* Saves the guest descriptor table registers and task register from the current
6854	* VMCS into the guest-CPU context.
6855	*
6856	* @returns VBox status code.
6857	* @param pVCpu The cross context virtual CPU structure.
6858	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6859	* out-of-sync. Make sure to update the required fields
6860	* before using them.
6861	*
6862	* @remarks No-long-jump zone!!!
6863	*/
6864	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6865	{
6866	int rc = VINF_SUCCESS;
6867
6868	/* Guest LDTR. */
6869	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6870	{
6871	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6872	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6873	AssertRCReturn(rc, rc);
6874	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6875	}
6876
6877	/* Guest GDTR. */
6878	uint64_t u64Val = 0;
6879	uint32_t u32Val = 0;
6880	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6881	{
6882	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6883	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6884	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6885	pMixedCtx->gdtr.pGdt = u64Val;
6886	pMixedCtx->gdtr.cbGdt = u32Val;
6887	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6888	}
6889
6890	/* Guest IDTR. */
6891	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6892	{
6893	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6894	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6895	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6896	pMixedCtx->idtr.pIdt = u64Val;
6897	pMixedCtx->idtr.cbIdt = u32Val;
6898	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6899	}
6900
6901	/* Guest TR. */
6902	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6903	{
6904	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6905	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6906	AssertRCReturn(rc, rc);
6907
6908	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6909	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6910	{
6911	rc = VMXLOCAL_READ_SEG(TR, tr);
6912	AssertRCReturn(rc, rc);
6913	}
6914	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6915	}
6916	return rc;
6917	}
6918
6919	#undef VMXLOCAL_READ_SEG
6920
6921
6922	/**
6923	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6924	* context.
6925	*
6926	* @returns VBox status code.
6927	* @param pVCpu The cross context virtual CPU structure.
6928	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6929	* out-of-sync. Make sure to update the required fields
6930	* before using them.
6931	*
6932	* @remarks No-long-jump zone!!!
6933	*/
6934	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6935	{
6936	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6937	{
6938	if (!pVCpu->hm.s.fUsingHyperDR7)
6939	{
6940	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6941	uint32_t u32Val;
6942	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6943	pMixedCtx->dr[7] = u32Val;
6944	}
6945
6946	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6947	}
6948	return VINF_SUCCESS;
6949	}
6950
6951
6952	/**
6953	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6954	*
6955	* @returns VBox status code.
6956	* @param pVCpu The cross context virtual CPU structure.
6957	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6958	* out-of-sync. Make sure to update the required fields
6959	* before using them.
6960	*
6961	* @remarks No-long-jump zone!!!
6962	*/
6963	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6964	{
6965	NOREF(pMixedCtx);
6966
6967	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6968	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6969	return VINF_SUCCESS;
6970	}
6971
6972
6973	/**
6974	* Saves the entire guest state from the currently active VMCS into the
6975	* guest-CPU context.
6976	*
6977	* This essentially VMREADs all guest-data.
6978	*
6979	* @returns VBox status code.
6980	* @param pVCpu The cross context virtual CPU structure.
6981	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6982	* out-of-sync. Make sure to update the required fields
6983	* before using them.
6984	*/
6985	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6986	{
6987	Assert(pVCpu);
6988	Assert(pMixedCtx);
6989
6990	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6991	return VINF_SUCCESS;
6992
6993	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6994	again on the ring-3 callback path, there is no real need to. */
6995	if (VMMRZCallRing3IsEnabled(pVCpu))
6996	VMMR0LogFlushDisable(pVCpu);
6997	else
6998	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6999	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7000
7001	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7002	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7003
7004	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7005	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7006
7007	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7008	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7009
7010	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7011	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7012
7013	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7014	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7015
7016	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7017	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7018
7019	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7020	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7021
7022	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7023	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7024
7025	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7026	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7027
7028	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7029	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7030
7031	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7032	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7033	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7034
7035	if (VMMRZCallRing3IsEnabled(pVCpu))
7036	VMMR0LogFlushEnable(pVCpu);
7037
7038	return VINF_SUCCESS;
7039	}
7040
7041
7042	/**
7043	* Saves basic guest registers needed for IEM instruction execution.
7044	*
7045	* @returns VBox status code (OR-able).
7046	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7047	* @param pMixedCtx Pointer to the CPU context of the guest.
7048	* @param fMemory Whether the instruction being executed operates on
7049	* memory or not. Only CR0 is synced up if clear.
7050	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7051	*/
7052	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7053	{
7054	/*
7055	* We assume all general purpose registers other than RSP are available.
7056	*
7057	* - RIP is a must, as it will be incremented or otherwise changed.
7058	* - RFLAGS are always required to figure the CPL.
7059	* - RSP isn't always required, however it's a GPR, so frequently required.
7060	* - SS and CS are the only segment register needed if IEM doesn't do memory
7061	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7062	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7063	* be required for memory accesses.
7064	*
7065	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7066	*/
7067	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7068	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7069	if (fNeedRsp)
7070	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7071	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7072	if (!fMemory)
7073	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7074	else
7075	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7076	AssertRCReturn(rc, rc);
7077	return rc;
7078	}
7079
7080
7081	/**
7082	* Ensures that we've got a complete basic guest-context.
7083	*
7084	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7085	* is for the interpreter.
7086	*
7087	* @returns VBox status code.
7088	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7089	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7090	* needing to be synced in.
7091	* @thread EMT(pVCpu)
7092	*/
7093	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7094	{
7095	/* Note! Since this is only applicable to VT-x, the implementation is placed
7096	in the VT-x part of the sources instead of the generic stuff. */
7097	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7098	{
7099	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7100	/*
7101	* For now, imply that the caller might change everything too. Do this after
7102	* saving the guest state so as to not trigger assertions.
7103	*/
7104	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7105	return rc;
7106	}
7107	return VINF_SUCCESS;
7108	}
7109
7110
7111	/**
7112	* Check per-VM and per-VCPU force flag actions that require us to go back to
7113	* ring-3 for one reason or another.
7114	*
7115	* @returns Strict VBox status code (i.e. informational status codes too)
7116	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7117	* ring-3.
7118	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7119	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7120	* interrupts)
7121	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7122	* all EMTs to be in ring-3.
7123	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7124	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7125	* to the EM loop.
7126	*
7127	* @param pVM The cross context VM structure.
7128	* @param pVCpu The cross context virtual CPU structure.
7129	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7130	* out-of-sync. Make sure to update the required fields
7131	* before using them.
7132	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7133	*/
7134	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7135	{
7136	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7137
7138	/*
7139	* Anything pending? Should be more likely than not if we're doing a good job.
7140	*/
7141	if ( !fStepping
7142	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7143	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7144	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7145	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7146	return VINF_SUCCESS;
7147
7148	/* We need the control registers now, make sure the guest-CPU context is updated. */
7149	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7150	AssertRCReturn(rc3, rc3);
7151
7152	/* Pending HM CR3 sync. */
7153	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7154	{
7155	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7156	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7157	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7158	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7159	}
7160
7161	/* Pending HM PAE PDPEs. */
7162	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7163	{
7164	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7165	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7166	}
7167
7168	/* Pending PGM C3 sync. */
7169	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7170	{
7171	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7172	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7173	if (rcStrict2 != VINF_SUCCESS)
7174	{
7175	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7176	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7177	return rcStrict2;
7178	}
7179	}
7180
7181	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7182	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7183	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7184	{
7185	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7186	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7187	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7188	return rc2;
7189	}
7190
7191	/* Pending VM request packets, such as hardware interrupts. */
7192	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7193	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7194	{
7195	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7196	return VINF_EM_PENDING_REQUEST;
7197	}
7198
7199	/* Pending PGM pool flushes. */
7200	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7201	{
7202	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7203	return VINF_PGM_POOL_FLUSH_PENDING;
7204	}
7205
7206	/* Pending DMA requests. */
7207	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7208	{
7209	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7210	return VINF_EM_RAW_TO_R3;
7211	}
7212
7213	return VINF_SUCCESS;
7214	}
7215
7216
7217	/**
7218	* Converts any TRPM trap into a pending HM event. This is typically used when
7219	* entering from ring-3 (not longjmp returns).
7220	*
7221	* @param pVCpu The cross context virtual CPU structure.
7222	*/
7223	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7224	{
7225	Assert(TRPMHasTrap(pVCpu));
7226	Assert(!pVCpu->hm.s.Event.fPending);
7227
7228	uint8_t uVector;
7229	TRPMEVENT enmTrpmEvent;
7230	RTGCUINT uErrCode;
7231	RTGCUINTPTR GCPtrFaultAddress;
7232	uint8_t cbInstr;
7233
7234	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7235	AssertRC(rc);
7236
7237	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7238	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7239	if (enmTrpmEvent == TRPM_TRAP)
7240	{
7241	switch (uVector)
7242	{
7243	case X86_XCPT_NMI:
7244	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7245	break;
7246
7247	case X86_XCPT_BP:
7248	case X86_XCPT_OF:
7249	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7250	break;
7251
7252	case X86_XCPT_PF:
7253	case X86_XCPT_DF:
7254	case X86_XCPT_TS:
7255	case X86_XCPT_NP:
7256	case X86_XCPT_SS:
7257	case X86_XCPT_GP:
7258	case X86_XCPT_AC:
7259	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7260	/* fall thru */
7261	default:
7262	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7263	break;
7264	}
7265	}
7266	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7267	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7268	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7269	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7270	else
7271	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7272
7273	rc = TRPMResetTrap(pVCpu);
7274	AssertRC(rc);
7275	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7276	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7277
7278	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7279	}
7280
7281
7282	/**
7283	* Converts the pending HM event into a TRPM trap.
7284	*
7285	* @param pVCpu The cross context virtual CPU structure.
7286	*/
7287	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7288	{
7289	Assert(pVCpu->hm.s.Event.fPending);
7290
7291	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7292	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7293	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7294	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7295
7296	/* If a trap was already pending, we did something wrong! */
7297	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7298
7299	TRPMEVENT enmTrapType;
7300	switch (uVectorType)
7301	{
7302	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7303	enmTrapType = TRPM_HARDWARE_INT;
7304	break;
7305
7306	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7307	enmTrapType = TRPM_SOFTWARE_INT;
7308	break;
7309
7310	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7311	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7312	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7313	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7314	enmTrapType = TRPM_TRAP;
7315	break;
7316
7317	default:
7318	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7319	enmTrapType = TRPM_32BIT_HACK;
7320	break;
7321	}
7322
7323	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7324
7325	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7326	AssertRC(rc);
7327
7328	if (fErrorCodeValid)
7329	TRPMSetErrorCode(pVCpu, uErrorCode);
7330
7331	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7332	&& uVector == X86_XCPT_PF)
7333	{
7334	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7335	}
7336	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7337	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7338	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7339	{
7340	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7341	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7342	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7343	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7344	}
7345
7346	/* Clear any pending events from the VMCS. */
7347	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7348	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7349
7350	/* We're now done converting the pending event. */
7351	pVCpu->hm.s.Event.fPending = false;
7352	}
7353
7354
7355	/**
7356	* Does the necessary state syncing before returning to ring-3 for any reason
7357	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7358	*
7359	* @returns VBox status code.
7360	* @param pVCpu The cross context virtual CPU structure.
7361	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7362	* be out-of-sync. Make sure to update the required
7363	* fields before using them.
7364	* @param fSaveGuestState Whether to save the guest state or not.
7365	*
7366	* @remarks No-long-jmp zone!!!
7367	*/
7368	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7369	{
7370	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7371	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7372
7373	RTCPUID idCpu = RTMpCpuId();
7374	Log4Func(("HostCpuId=%u\n", idCpu));
7375
7376	/*
7377	* !!! IMPORTANT !!!
7378	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7379	*/
7380
7381	/* Save the guest state if necessary. */
7382	if ( fSaveGuestState
7383	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7384	{
7385	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7386	AssertRCReturn(rc, rc);
7387	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7388	}
7389
7390	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7391	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7392	{
7393	/* We shouldn't reload CR0 without saving it first. */
7394	if (!fSaveGuestState)
7395	{
7396	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7397	AssertRCReturn(rc, rc);
7398	}
7399	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7400	}
7401
7402	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7403	#ifdef VBOX_STRICT
7404	if (CPUMIsHyperDebugStateActive(pVCpu))
7405	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7406	#endif
7407	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7408	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7409	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7410	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7411
7412	#if HC_ARCH_BITS == 64
7413	/* Restore host-state bits that VT-x only restores partially. */
7414	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7415	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7416	{
7417	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7418	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7419	}
7420	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7421	#endif
7422
7423	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7424	if (pVCpu->hm.s.vmx.fLazyMsrs)
7425	{
7426	/* We shouldn't reload the guest MSRs without saving it first. */
7427	if (!fSaveGuestState)
7428	{
7429	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7430	AssertRCReturn(rc, rc);
7431	}
7432	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7433	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7434	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7435	}
7436
7437	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7438	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7439
7440	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7441	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7442	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7443	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7444	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7445	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7446	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7447	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7448
7449	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7450
7451	/** @todo This partially defeats the purpose of having preemption hooks.
7452	* The problem is, deregistering the hooks should be moved to a place that
7453	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7454	* context.
7455	*/
7456	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7457	{
7458	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7459	AssertRCReturn(rc, rc);
7460
7461	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7462	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7463	}
7464	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7465	NOREF(idCpu);
7466
7467	return VINF_SUCCESS;
7468	}
7469
7470
7471	/**
7472	* Leaves the VT-x session.
7473	*
7474	* @returns VBox status code.
7475	* @param pVCpu The cross context virtual CPU structure.
7476	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7477	* out-of-sync. Make sure to update the required fields
7478	* before using them.
7479	*
7480	* @remarks No-long-jmp zone!!!
7481	*/
7482	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7483	{
7484	HM_DISABLE_PREEMPT();
7485	HMVMX_ASSERT_CPU_SAFE();
7486	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7487	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7488
7489	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7490	and done this from the VMXR0ThreadCtxCallback(). */
7491	if (!pVCpu->hm.s.fLeaveDone)
7492	{
7493	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7494	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7495	pVCpu->hm.s.fLeaveDone = true;
7496	}
7497	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7498
7499	/*
7500	* !!! IMPORTANT !!!
7501	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7502	*/
7503
7504	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7505	/** @todo Deregistering here means we need to VMCLEAR always
7506	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7507	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7508	VMMR0ThreadCtxHookDisable(pVCpu);
7509
7510	/* Leave HM context. This takes care of local init (term). */
7511	int rc = HMR0LeaveCpu(pVCpu);
7512
7513	HM_RESTORE_PREEMPT();
7514	return rc;
7515	}
7516
7517
7518	/**
7519	* Does the necessary state syncing before doing a longjmp to ring-3.
7520	*
7521	* @returns VBox status code.
7522	* @param pVCpu The cross context virtual CPU structure.
7523	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7524	* out-of-sync. Make sure to update the required fields
7525	* before using them.
7526	*
7527	* @remarks No-long-jmp zone!!!
7528	*/
7529	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7530	{
7531	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7532	}
7533
7534
7535	/**
7536	* Take necessary actions before going back to ring-3.
7537	*
7538	* An action requires us to go back to ring-3. This function does the necessary
7539	* steps before we can safely return to ring-3. This is not the same as longjmps
7540	* to ring-3, this is voluntary and prepares the guest so it may continue
7541	* executing outside HM (recompiler/IEM).
7542	*
7543	* @returns VBox status code.
7544	* @param pVM The cross context VM structure.
7545	* @param pVCpu The cross context virtual CPU structure.
7546	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7547	* out-of-sync. Make sure to update the required fields
7548	* before using them.
7549	* @param rcExit The reason for exiting to ring-3. Can be
7550	* VINF_VMM_UNKNOWN_RING3_CALL.
7551	*/
7552	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7553	{
7554	Assert(pVM);
7555	Assert(pVCpu);
7556	Assert(pMixedCtx);
7557	HMVMX_ASSERT_PREEMPT_SAFE();
7558
7559	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7560	{
7561	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7562	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7563	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7564	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7565	}
7566
7567	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7568	VMMRZCallRing3Disable(pVCpu);
7569	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7570
7571	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7572	if (pVCpu->hm.s.Event.fPending)
7573	{
7574	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7575	Assert(!pVCpu->hm.s.Event.fPending);
7576	}
7577
7578	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7579	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7580
7581	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7582	and if we're injecting an event we should have a TRPM trap pending. */
7583	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7584	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7585	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7586	#endif
7587
7588	/* Save guest state and restore host state bits. */
7589	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7590	AssertRCReturn(rc, rc);
7591	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7592	/* Thread-context hooks are unregistered at this point!!! */
7593
7594	/* Sync recompiler state. */
7595	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7596	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7597	\| CPUM_CHANGED_LDTR
7598	\| CPUM_CHANGED_GDTR
7599	\| CPUM_CHANGED_IDTR
7600	\| CPUM_CHANGED_TR
7601	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7602	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7603	if ( pVM->hm.s.fNestedPaging
7604	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7605	{
7606	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7607	}
7608
7609	Assert(!pVCpu->hm.s.fClearTrapFlag);
7610
7611	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7612	if (rcExit != VINF_EM_RAW_INTERRUPT)
7613	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7614
7615	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7616
7617	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7618	VMMRZCallRing3RemoveNotification(pVCpu);
7619	VMMRZCallRing3Enable(pVCpu);
7620
7621	return rc;
7622	}
7623
7624
7625	/**
7626	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7627	* longjump to ring-3 and possibly get preempted.
7628	*
7629	* @returns VBox status code.
7630	* @param pVCpu The cross context virtual CPU structure.
7631	* @param enmOperation The operation causing the ring-3 longjump.
7632	* @param pvUser Opaque pointer to the guest-CPU context. The data
7633	* may be out-of-sync. Make sure to update the required
7634	* fields before using them.
7635	*/
7636	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7637	{
7638	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7639	{
7640	/*
7641	* !!! IMPORTANT !!!
7642	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7643	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7644	*/
7645	VMMRZCallRing3RemoveNotification(pVCpu);
7646	VMMRZCallRing3Disable(pVCpu);
7647	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7648	RTThreadPreemptDisable(&PreemptState);
7649
7650	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7651	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7652
7653	#if HC_ARCH_BITS == 64
7654	/* Restore host-state bits that VT-x only restores partially. */
7655	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7656	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7657	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7658	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7659	#endif
7660	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7661	if (pVCpu->hm.s.vmx.fLazyMsrs)
7662	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7663
7664	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7665	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7666	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7667	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7668	{
7669	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7670	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7671	}
7672
7673	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7674	VMMR0ThreadCtxHookDisable(pVCpu);
7675	HMR0LeaveCpu(pVCpu);
7676	RTThreadPreemptRestore(&PreemptState);
7677	return VINF_SUCCESS;
7678	}
7679
7680	Assert(pVCpu);
7681	Assert(pvUser);
7682	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7683	HMVMX_ASSERT_PREEMPT_SAFE();
7684
7685	VMMRZCallRing3Disable(pVCpu);
7686	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7687
7688	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7689	enmOperation));
7690
7691	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7692	AssertRCReturn(rc, rc);
7693
7694	VMMRZCallRing3Enable(pVCpu);
7695	return VINF_SUCCESS;
7696	}
7697
7698
7699	/**
7700	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7701	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7702	*
7703	* @param pVCpu The cross context virtual CPU structure.
7704	*/
7705	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7706	{
7707	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7708	{
7709	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7710	{
7711	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7712	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7713	AssertRC(rc);
7714	Log4(("Setup interrupt-window exiting\n"));
7715	}
7716	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7717	}
7718
7719
7720	/**
7721	* Clears the interrupt-window exiting control in the VMCS.
7722	*
7723	* @param pVCpu The cross context virtual CPU structure.
7724	*/
7725	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7726	{
7727	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7728	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7729	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7730	AssertRC(rc);
7731	Log4(("Cleared interrupt-window exiting\n"));
7732	}
7733
7734
7735	/**
7736	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7737	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7738	*
7739	* @param pVCpu The cross context virtual CPU structure.
7740	*/
7741	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7742	{
7743	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7744	{
7745	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7746	{
7747	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7748	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7749	AssertRC(rc);
7750	Log4(("Setup NMI-window exiting\n"));
7751	}
7752	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7753	}
7754
7755
7756	/**
7757	* Clears the NMI-window exiting control in the VMCS.
7758	*
7759	* @param pVCpu The cross context virtual CPU structure.
7760	*/
7761	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7762	{
7763	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7764	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7765	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7766	AssertRC(rc);
7767	Log4(("Cleared NMI-window exiting\n"));
7768	}
7769
7770
7771	/**
7772	* Evaluates the event to be delivered to the guest and sets it as the pending
7773	* event.
7774	*
7775	* @returns The VT-x guest-interruptibility state.
7776	* @param pVCpu The cross context virtual CPU structure.
7777	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7778	* out-of-sync. Make sure to update the required fields
7779	* before using them.
7780	*/
7781	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7782	{
7783	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7784	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7785	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7786	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7787	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7788
7789	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7790	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7791	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7792	Assert(!TRPMHasTrap(pVCpu));
7793
7794	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7795	APICUpdatePendingInterrupts(pVCpu);
7796
7797	/*
7798	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7799	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7800	*/
7801	/** @todo SMI. SMIs take priority over NMIs. */
7802	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7803	{
7804	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7805	if ( !pVCpu->hm.s.Event.fPending
7806	&& !fBlockNmi
7807	&& !fBlockSti
7808	&& !fBlockMovSS)
7809	{
7810	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7811	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7812	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7813
7814	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7815	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7816	}
7817	else
7818	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7819	}
7820	/*
7821	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7822	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7823	*/
7824	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7825	&& !pVCpu->hm.s.fSingleInstruction)
7826	{
7827	Assert(!DBGFIsStepping(pVCpu));
7828	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7829	AssertRC(rc);
7830	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7831	if ( !pVCpu->hm.s.Event.fPending
7832	&& !fBlockInt
7833	&& !fBlockSti
7834	&& !fBlockMovSS)
7835	{
7836	uint8_t u8Interrupt;
7837	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7838	if (RT_SUCCESS(rc))
7839	{
7840	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7841	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7842	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7843
7844	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7845	}
7846	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7847	{
7848	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7849	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7850	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7851
7852	/*
7853	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7854	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7855	* need to re-set this force-flag here.
7856	*/
7857	}
7858	else
7859	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7860	}
7861	else
7862	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7863	}
7864
7865	return uIntrState;
7866	}
7867
7868
7869	/**
7870	* Sets a pending-debug exception to be delivered to the guest if the guest is
7871	* single-stepping in the VMCS.
7872	*
7873	* @param pVCpu The cross context virtual CPU structure.
7874	*/
7875	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7876	{
7877	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7878	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7879	AssertRC(rc);
7880	}
7881
7882
7883	/**
7884	* Injects any pending events into the guest if the guest is in a state to
7885	* receive them.
7886	*
7887	* @returns Strict VBox status code (i.e. informational status codes too).
7888	* @param pVCpu The cross context virtual CPU structure.
7889	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7890	* out-of-sync. Make sure to update the required fields
7891	* before using them.
7892	* @param uIntrState The VT-x guest-interruptibility state.
7893	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7894	* return VINF_EM_DBG_STEPPED if the event was
7895	* dispatched directly.
7896	*/
7897	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7898	{
7899	HMVMX_ASSERT_PREEMPT_SAFE();
7900	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7901
7902	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7903	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7904
7905	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7906	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7907	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7908	Assert(!TRPMHasTrap(pVCpu));
7909
7910	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7911	if (pVCpu->hm.s.Event.fPending)
7912	{
7913	/*
7914	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7915	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7916	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7917	*
7918	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7919	*/
7920	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7921	#ifdef VBOX_STRICT
7922	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7923	{
7924	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7925	Assert(!fBlockInt);
7926	Assert(!fBlockSti);
7927	Assert(!fBlockMovSS);
7928	}
7929	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7930	{
7931	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7932	Assert(!fBlockSti);
7933	Assert(!fBlockMovSS);
7934	Assert(!fBlockNmi);
7935	}
7936	#endif
7937	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7938	(uint8_t)uIntType));
7939	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7940	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7941	fStepping, &uIntrState);
7942	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7943
7944	/* Update the interruptibility-state as it could have been changed by
7945	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7946	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7947	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7948
7949	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7950	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7951	else
7952	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7953	}
7954
7955	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7956	if ( fBlockSti
7957	\|\| fBlockMovSS)
7958	{
7959	if (!pVCpu->hm.s.fSingleInstruction)
7960	{
7961	/*
7962	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7963	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7964	* See Intel spec. 27.3.4 "Saving Non-Register State".
7965	*/
7966	Assert(!DBGFIsStepping(pVCpu));
7967	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7968	AssertRCReturn(rc2, rc2);
7969	if (pMixedCtx->eflags.Bits.u1TF)
7970	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7971	}
7972	else if (pMixedCtx->eflags.Bits.u1TF)
7973	{
7974	/*
7975	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7976	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7977	*/
7978	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7979	uIntrState = 0;
7980	}
7981	}
7982
7983	/*
7984	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7985	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7986	*/
7987	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7988	AssertRC(rc2);
7989
7990	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7991	NOREF(fBlockMovSS); NOREF(fBlockSti);
7992	return rcStrict;
7993	}
7994
7995
7996	/**
7997	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7998	*
7999	* @param pVCpu The cross context virtual CPU structure.
8000	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8001	* out-of-sync. Make sure to update the required fields
8002	* before using them.
8003	*/
8004	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8005	{
8006	NOREF(pMixedCtx);
8007	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8008	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8009	}
8010
8011
8012	/**
8013	* Injects a double-fault (\#DF) exception into the VM.
8014	*
8015	* @returns Strict VBox status code (i.e. informational status codes too).
8016	* @param pVCpu The cross context virtual CPU structure.
8017	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8018	* out-of-sync. Make sure to update the required fields
8019	* before using them.
8020	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8021	* and should return VINF_EM_DBG_STEPPED if the event
8022	* is injected directly (register modified by us, not
8023	* by hardware on VM-entry).
8024	* @param puIntrState Pointer to the current guest interruptibility-state.
8025	* This interruptibility-state will be updated if
8026	* necessary. This cannot not be NULL.
8027	*/
8028	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8029	{
8030	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8031	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8032	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8033	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8034	fStepping, puIntrState);
8035	}
8036
8037
8038	/**
8039	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8040	*
8041	* @param pVCpu The cross context virtual CPU structure.
8042	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8043	* out-of-sync. Make sure to update the required fields
8044	* before using them.
8045	*/
8046	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8047	{
8048	NOREF(pMixedCtx);
8049	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8050	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8051	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8052	}
8053
8054
8055	/**
8056	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8057	*
8058	* @param pVCpu The cross context virtual CPU structure.
8059	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8060	* out-of-sync. Make sure to update the required fields
8061	* before using them.
8062	* @param cbInstr The value of RIP that is to be pushed on the guest
8063	* stack.
8064	*/
8065	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8066	{
8067	NOREF(pMixedCtx);
8068	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8069	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8070	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8071	}
8072
8073
8074	/**
8075	* Injects a general-protection (\#GP) fault into the VM.
8076	*
8077	* @returns Strict VBox status code (i.e. informational status codes too).
8078	* @param pVCpu The cross context virtual CPU structure.
8079	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8080	* out-of-sync. Make sure to update the required fields
8081	* before using them.
8082	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8083	* mode, i.e. in real-mode it's not valid).
8084	* @param u32ErrorCode The error code associated with the \#GP.
8085	* @param fStepping Whether we're running in
8086	* hmR0VmxRunGuestCodeStep() and should return
8087	* VINF_EM_DBG_STEPPED if the event is injected
8088	* directly (register modified by us, not by
8089	* hardware on VM-entry).
8090	* @param puIntrState Pointer to the current guest interruptibility-state.
8091	* This interruptibility-state will be updated if
8092	* necessary. This cannot not be NULL.
8093	*/
8094	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8095	bool fStepping, uint32_t *puIntrState)
8096	{
8097	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8098	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8099	if (fErrorCodeValid)
8100	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8101	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8102	fStepping, puIntrState);
8103	}
8104
8105
8106	#if 0 /* unused */
8107	/**
8108	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8109	* VM.
8110	*
8111	* @param pVCpu The cross context virtual CPU structure.
8112	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8113	* out-of-sync. Make sure to update the required fields
8114	* before using them.
8115	* @param u32ErrorCode The error code associated with the \#GP.
8116	*/
8117	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8118	{
8119	NOREF(pMixedCtx);
8120	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8121	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8122	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8123	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8124	}
8125	#endif /* unused */
8126
8127
8128	/**
8129	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8130	*
8131	* @param pVCpu The cross context virtual CPU structure.
8132	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8133	* out-of-sync. Make sure to update the required fields
8134	* before using them.
8135	* @param uVector The software interrupt vector number.
8136	* @param cbInstr The value of RIP that is to be pushed on the guest
8137	* stack.
8138	*/
8139	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8140	{
8141	NOREF(pMixedCtx);
8142	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8143	if ( uVector == X86_XCPT_BP
8144	\|\| uVector == X86_XCPT_OF)
8145	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8146	else
8147	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8148	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8149	}
8150
8151
8152	/**
8153	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8154	* stack.
8155	*
8156	* @returns Strict VBox status code (i.e. informational status codes too).
8157	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8158	* @param pVM The cross context VM structure.
8159	* @param pMixedCtx Pointer to the guest-CPU context.
8160	* @param uValue The value to push to the guest stack.
8161	*/
8162	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8163	{
8164	/*
8165	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8166	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8167	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8168	*/
8169	if (pMixedCtx->sp == 1)
8170	return VINF_EM_RESET;
8171	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8172	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8173	AssertRC(rc);
8174	return rc;
8175	}
8176
8177
8178	/**
8179	* Injects an event into the guest upon VM-entry by updating the relevant fields
8180	* in the VM-entry area in the VMCS.
8181	*
8182	* @returns Strict VBox status code (i.e. informational status codes too).
8183	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8184	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8185	*
8186	* @param pVCpu The cross context virtual CPU structure.
8187	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8188	* be out-of-sync. Make sure to update the required
8189	* fields before using them.
8190	* @param u64IntInfo The VM-entry interruption-information field.
8191	* @param cbInstr The VM-entry instruction length in bytes (for
8192	* software interrupts, exceptions and privileged
8193	* software exceptions).
8194	* @param u32ErrCode The VM-entry exception error code.
8195	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8196	* @param puIntrState Pointer to the current guest interruptibility-state.
8197	* This interruptibility-state will be updated if
8198	* necessary. This cannot not be NULL.
8199	* @param fStepping Whether we're running in
8200	* hmR0VmxRunGuestCodeStep() and should return
8201	* VINF_EM_DBG_STEPPED if the event is injected
8202	* directly (register modified by us, not by
8203	* hardware on VM-entry).
8204	*
8205	* @remarks Requires CR0!
8206	*/
8207	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8208	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8209	uint32_t *puIntrState)
8210	{
8211	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8212	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8213	Assert(puIntrState);
8214	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8215
8216	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8217	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8218
8219	#ifdef VBOX_STRICT
8220	/* Validate the error-code-valid bit for hardware exceptions. */
8221	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8222	{
8223	switch (uVector)
8224	{
8225	case X86_XCPT_PF:
8226	case X86_XCPT_DF:
8227	case X86_XCPT_TS:
8228	case X86_XCPT_NP:
8229	case X86_XCPT_SS:
8230	case X86_XCPT_GP:
8231	case X86_XCPT_AC:
8232	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8233	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8234	/* fall thru */
8235	default:
8236	break;
8237	}
8238	}
8239	#endif
8240
8241	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8242	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8243	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8244
8245	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8246
8247	/* We require CR0 to check if the guest is in real-mode. */
8248	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8249	AssertRCReturn(rc, rc);
8250
8251	/*
8252	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8253	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8254	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8255	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8256	*/
8257	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8258	{
8259	PVM pVM = pVCpu->CTX_SUFF(pVM);
8260	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8261	{
8262	Assert(PDMVmmDevHeapIsEnabled(pVM));
8263	Assert(pVM->hm.s.vmx.pRealModeTSS);
8264
8265	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8266	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8267	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8268	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8269	AssertRCReturn(rc, rc);
8270	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8271
8272	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8273	size_t const cbIdtEntry = sizeof(X86IDTR16);
8274	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8275	{
8276	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8277	if (uVector == X86_XCPT_DF)
8278	return VINF_EM_RESET;
8279
8280	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8281	if (uVector == X86_XCPT_GP)
8282	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8283
8284	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8285	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8286	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8287	fStepping, puIntrState);
8288	}
8289
8290	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8291	uint16_t uGuestIp = pMixedCtx->ip;
8292	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8293	{
8294	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8295	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8296	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8297	}
8298	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8299	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8300
8301	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8302	X86IDTR16 IdtEntry;
8303	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8304	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8305	AssertRCReturn(rc, rc);
8306
8307	/* Construct the stack frame for the interrupt/exception handler. */
8308	VBOXSTRICTRC rcStrict;
8309	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8310	if (rcStrict == VINF_SUCCESS)
8311	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8312	if (rcStrict == VINF_SUCCESS)
8313	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8314
8315	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8316	if (rcStrict == VINF_SUCCESS)
8317	{
8318	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8319	pMixedCtx->rip = IdtEntry.offSel;
8320	pMixedCtx->cs.Sel = IdtEntry.uSel;
8321	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8322	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8323	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8324	&& uVector == X86_XCPT_PF)
8325	pMixedCtx->cr2 = GCPtrFaultAddress;
8326
8327	/* If any other guest-state bits are changed here, make sure to update
8328	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8329	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8330	\| HM_CHANGED_GUEST_RIP
8331	\| HM_CHANGED_GUEST_RFLAGS
8332	\| HM_CHANGED_GUEST_RSP);
8333
8334	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8335	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8336	{
8337	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8338	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8339	Log4(("Clearing inhibition due to STI.\n"));
8340	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8341	}
8342	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8343	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8344
8345	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8346	it, if we are returning to ring-3 before executing guest code. */
8347	pVCpu->hm.s.Event.fPending = false;
8348
8349	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8350	if (fStepping)
8351	rcStrict = VINF_EM_DBG_STEPPED;
8352	}
8353	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8354	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8355	return rcStrict;
8356	}
8357
8358	/*
8359	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8360	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8361	*/
8362	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8363	}
8364
8365	/* Validate. */
8366	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8367	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8368	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8369
8370	/* Inject. */
8371	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8372	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8373	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8374	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8375
8376	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8377	&& uVector == X86_XCPT_PF)
8378	pMixedCtx->cr2 = GCPtrFaultAddress;
8379
8380	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8381	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8382
8383	AssertRCReturn(rc, rc);
8384	return VINF_SUCCESS;
8385	}
8386
8387
8388	/**
8389	* Clears the interrupt-window exiting control in the VMCS and if necessary
8390	* clears the current event in the VMCS as well.
8391	*
8392	* @returns VBox status code.
8393	* @param pVCpu The cross context virtual CPU structure.
8394	*
8395	* @remarks Use this function only to clear events that have not yet been
8396	* delivered to the guest but are injected in the VMCS!
8397	* @remarks No-long-jump zone!!!
8398	*/
8399	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8400	{
8401	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8402
8403	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8404	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8405
8406	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8407	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8408	}
8409
8410
8411	/**
8412	* Enters the VT-x session.
8413	*
8414	* @returns VBox status code.
8415	* @param pVM The cross context VM structure.
8416	* @param pVCpu The cross context virtual CPU structure.
8417	* @param pCpu Pointer to the CPU info struct.
8418	*/
8419	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8420	{
8421	AssertPtr(pVM);
8422	AssertPtr(pVCpu);
8423	Assert(pVM->hm.s.vmx.fSupported);
8424	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8425	NOREF(pCpu); NOREF(pVM);
8426
8427	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8428	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8429
8430	#ifdef VBOX_STRICT
8431	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8432	RTCCUINTREG uHostCR4 = ASMGetCR4();
8433	if (!(uHostCR4 & X86_CR4_VMXE))
8434	{
8435	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8436	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8437	}
8438	#endif
8439
8440	/*
8441	* Load the VCPU's VMCS as the current (and active) one.
8442	*/
8443	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8444	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8445	if (RT_FAILURE(rc))
8446	return rc;
8447
8448	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8449	pVCpu->hm.s.fLeaveDone = false;
8450	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8451
8452	return VINF_SUCCESS;
8453	}
8454
8455
8456	/**
8457	* The thread-context callback (only on platforms which support it).
8458	*
8459	* @param enmEvent The thread-context event.
8460	* @param pVCpu The cross context virtual CPU structure.
8461	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8462	* @thread EMT(pVCpu)
8463	*/
8464	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8465	{
8466	NOREF(fGlobalInit);
8467
8468	switch (enmEvent)
8469	{
8470	case RTTHREADCTXEVENT_OUT:
8471	{
8472	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8473	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8474	VMCPU_ASSERT_EMT(pVCpu);
8475
8476	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8477
8478	/* No longjmps (logger flushes, locks) in this fragile context. */
8479	VMMRZCallRing3Disable(pVCpu);
8480	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8481
8482	/*
8483	* Restore host-state (FPU, debug etc.)
8484	*/
8485	if (!pVCpu->hm.s.fLeaveDone)
8486	{
8487	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8488	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8489	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8490	pVCpu->hm.s.fLeaveDone = true;
8491	}
8492
8493	/* Leave HM context, takes care of local init (term). */
8494	int rc = HMR0LeaveCpu(pVCpu);
8495	AssertRC(rc); NOREF(rc);
8496
8497	/* Restore longjmp state. */
8498	VMMRZCallRing3Enable(pVCpu);
8499	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8500	break;
8501	}
8502
8503	case RTTHREADCTXEVENT_IN:
8504	{
8505	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8506	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8507	VMCPU_ASSERT_EMT(pVCpu);
8508
8509	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8510	VMMRZCallRing3Disable(pVCpu);
8511	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8512
8513	/* Initialize the bare minimum state required for HM. This takes care of
8514	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8515	int rc = HMR0EnterCpu(pVCpu);
8516	AssertRC(rc);
8517	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8518
8519	/* Load the active VMCS as the current one. */
8520	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8521	{
8522	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8523	AssertRC(rc); NOREF(rc);
8524	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8525	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8526	}
8527	pVCpu->hm.s.fLeaveDone = false;
8528
8529	/* Restore longjmp state. */
8530	VMMRZCallRing3Enable(pVCpu);
8531	break;
8532	}
8533
8534	default:
8535	break;
8536	}
8537	}
8538
8539
8540	/**
8541	* Saves the host state in the VMCS host-state.
8542	* Sets up the VM-exit MSR-load area.
8543	*
8544	* The CPU state will be loaded from these fields on every successful VM-exit.
8545	*
8546	* @returns VBox status code.
8547	* @param pVM The cross context VM structure.
8548	* @param pVCpu The cross context virtual CPU structure.
8549	*
8550	* @remarks No-long-jump zone!!!
8551	*/
8552	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8553	{
8554	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8555
8556	int rc = VINF_SUCCESS;
8557	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8558	{
8559	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8560	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8561
8562	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8563	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8564
8565	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8566	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8567
8568	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8569	}
8570	return rc;
8571	}
8572
8573
8574	/**
8575	* Saves the host state in the VMCS host-state.
8576	*
8577	* @returns VBox status code.
8578	* @param pVM The cross context VM structure.
8579	* @param pVCpu The cross context virtual CPU structure.
8580	*
8581	* @remarks No-long-jump zone!!!
8582	*/
8583	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8584	{
8585	AssertPtr(pVM);
8586	AssertPtr(pVCpu);
8587
8588	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8589
8590	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8591	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8593	return hmR0VmxSaveHostState(pVM, pVCpu);
8594	}
8595
8596
8597	/**
8598	* Loads the guest state into the VMCS guest-state area.
8599	*
8600	* The will typically be done before VM-entry when the guest-CPU state and the
8601	* VMCS state may potentially be out of sync.
8602	*
8603	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8604	* VM-entry controls.
8605	* Sets up the appropriate VMX non-root function to execute guest code based on
8606	* the guest CPU mode.
8607	*
8608	* @returns VBox strict status code.
8609	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8610	* without unrestricted guest access and the VMMDev is not presently
8611	* mapped (e.g. EFI32).
8612	*
8613	* @param pVM The cross context VM structure.
8614	* @param pVCpu The cross context virtual CPU structure.
8615	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8616	* out-of-sync. Make sure to update the required fields
8617	* before using them.
8618	*
8619	* @remarks No-long-jump zone!!!
8620	*/
8621	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8622	{
8623	AssertPtr(pVM);
8624	AssertPtr(pVCpu);
8625	AssertPtr(pMixedCtx);
8626	HMVMX_ASSERT_PREEMPT_SAFE();
8627
8628	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8629
8630	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8631
8632	/* Determine real-on-v86 mode. */
8633	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8634	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8635	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8636	{
8637	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8638	}
8639
8640	/*
8641	* Load the guest-state into the VMCS.
8642	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8643	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8644	*/
8645	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8646	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8647
8648	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8649	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8650	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8651
8652	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8653	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8654	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8655
8656	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8657	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8658
8659	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8660	if (rcStrict == VINF_SUCCESS)
8661	{ /* likely */ }
8662	else
8663	{
8664	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8665	return rcStrict;
8666	}
8667
8668	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8669	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8670	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8671
8672	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8673	determine we don't have to swap EFER after all. */
8674	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8675	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8676
8677	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8678	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8679
8680	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8681	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8682
8683	/*
8684	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8685	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8686	*/
8687	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8688	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8689
8690	/* Clear any unused and reserved bits. */
8691	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8692
8693	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8694	return rc;
8695	}
8696
8697
8698	/**
8699	* Loads the state shared between the host and guest into the VMCS.
8700	*
8701	* @param pVM The cross context VM structure.
8702	* @param pVCpu The cross context virtual CPU structure.
8703	* @param pCtx Pointer to the guest-CPU context.
8704	*
8705	* @remarks No-long-jump zone!!!
8706	*/
8707	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8708	{
8709	NOREF(pVM);
8710
8711	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8712	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8713
8714	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8715	{
8716	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8717	AssertRC(rc);
8718	}
8719
8720	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8721	{
8722	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8723	AssertRC(rc);
8724
8725	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8726	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8727	{
8728	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8729	AssertRC(rc);
8730	}
8731	}
8732
8733	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8734	{
8735	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8736	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8737	}
8738
8739	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8740	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8741	{
8742	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8743	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8744	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8745	AssertRC(rc);
8746	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8747	}
8748
8749	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8750	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8751	}
8752
8753
8754	/**
8755	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8756	*
8757	* @returns Strict VBox status code (i.e. informational status codes too).
8758	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8759	* without unrestricted guest access and the VMMDev is not presently
8760	* mapped (e.g. EFI32).
8761	*
8762	* @param pVM The cross context VM structure.
8763	* @param pVCpu The cross context virtual CPU structure.
8764	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8765	* out-of-sync. Make sure to update the required fields
8766	* before using them.
8767	*
8768	* @remarks No-long-jump zone!!!
8769	*/
8770	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8771	{
8772	HMVMX_ASSERT_PREEMPT_SAFE();
8773	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8774	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8775
8776	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8777	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8778	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8779	#endif
8780
8781	/*
8782	* RIP is what changes the most often and hence if it's the only bit needing to be
8783	* updated, we shall handle it early for performance reasons.
8784	*/
8785	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8786	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8787	{
8788	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8789	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8790	{ /* likely */}
8791	else
8792	{
8793	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8794	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8795	}
8796	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8797	}
8798	else if (HMCPU_CF_VALUE(pVCpu))
8799	{
8800	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8801	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8802	{ /* likely */}
8803	else
8804	{
8805	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8806	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8807	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8808	return rcStrict;
8809	}
8810	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8811	}
8812
8813	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8814	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8815	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8816	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8817	return rcStrict;
8818	}
8819
8820
8821	/**
8822	* Does the preparations before executing guest code in VT-x.
8823	*
8824	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8825	* recompiler/IEM. We must be cautious what we do here regarding committing
8826	* guest-state information into the VMCS assuming we assuredly execute the
8827	* guest in VT-x mode.
8828	*
8829	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8830	* the common-state (TRPM/forceflags), we must undo those changes so that the
8831	* recompiler/IEM can (and should) use them when it resumes guest execution.
8832	* Otherwise such operations must be done when we can no longer exit to ring-3.
8833	*
8834	* @returns Strict VBox status code (i.e. informational status codes too).
8835	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8836	* have been disabled.
8837	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8838	* double-fault into the guest.
8839	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8840	* dispatched directly.
8841	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8842	*
8843	* @param pVM The cross context VM structure.
8844	* @param pVCpu The cross context virtual CPU structure.
8845	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8846	* out-of-sync. Make sure to update the required fields
8847	* before using them.
8848	* @param pVmxTransient Pointer to the VMX transient structure.
8849	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8850	* us ignore some of the reasons for returning to
8851	* ring-3, and return VINF_EM_DBG_STEPPED if event
8852	* dispatching took place.
8853	*/
8854	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8855	{
8856	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8857
8858	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8859	PGMRZDynMapFlushAutoSet(pVCpu);
8860	#endif
8861
8862	/* Check force flag actions that might require us to go back to ring-3. */
8863	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8864	if (rcStrict == VINF_SUCCESS)
8865	{ /* FFs doesn't get set all the time. */ }
8866	else
8867	return rcStrict;
8868
8869	#ifndef IEM_VERIFICATION_MODE_FULL
8870	/*
8871	* Setup the virtualized-APIC accesses.
8872	*
8873	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8874	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8875	*
8876	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8877	*/
8878	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8879	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8880	&& PDMHasApic(pVM))
8881	{
8882	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8883	Assert(u64MsrApicBase);
8884	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8885
8886	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8887
8888	/* Unalias any existing mapping. */
8889	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8890	AssertRCReturn(rc, rc);
8891
8892	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8893	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8894	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8895	AssertRCReturn(rc, rc);
8896
8897	/* Update the per-VCPU cache of the APIC base MSR. */
8898	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8899	}
8900	#endif /* !IEM_VERIFICATION_MODE_FULL */
8901
8902	if (TRPMHasTrap(pVCpu))
8903	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8904	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8905
8906	/*
8907	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8908	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8909	*/
8910	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8911	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8912	{ /* likely */ }
8913	else
8914	{
8915	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8916	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8917	return rcStrict;
8918	}
8919
8920	/*
8921	* No longjmps to ring-3 from this point on!!!
8922	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8923	* This also disables flushing of the R0-logger instance (if any).
8924	*/
8925	VMMRZCallRing3Disable(pVCpu);
8926
8927	/*
8928	* Load the guest state bits.
8929	*
8930	* We cannot perform longjmps while loading the guest state because we do not preserve the
8931	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8932	* CPU migration.
8933	*
8934	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8935	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8936	* Hence, loading of the guest state needs to be done -after- injection of events.
8937	*/
8938	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8939	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8940	{ /* likely */ }
8941	else
8942	{
8943	VMMRZCallRing3Enable(pVCpu);
8944	return rcStrict;
8945	}
8946
8947	/*
8948	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8949	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8950	*
8951	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8952	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8953	*
8954	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8955	* executing guest code.
8956	*/
8957	pVmxTransient->fEFlags = ASMIntDisableFlags();
8958
8959	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8960	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8961	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8962	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8963	{
8964	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8965	{
8966	pVCpu->hm.s.Event.fPending = false;
8967
8968	/*
8969	* We've injected any pending events. This is really the point of no return (to ring-3).
8970	*
8971	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
8972	* returns from this function, so don't enable them here.
8973	*/
8974	return VINF_SUCCESS;
8975	}
8976
8977	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8978	rcStrict = VINF_EM_RAW_INTERRUPT;
8979	}
8980	else
8981	{
8982	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8983	rcStrict = VINF_EM_RAW_TO_R3;
8984	}
8985
8986	ASMSetFlags(pVmxTransient->fEFlags);
8987	VMMRZCallRing3Enable(pVCpu);
8988
8989	return rcStrict;
8990	}
8991
8992
8993	/**
8994	* Prepares to run guest code in VT-x and we've committed to doing so. This
8995	* means there is no backing out to ring-3 or anywhere else at this
8996	* point.
8997	*
8998	* @param pVM The cross context VM structure.
8999	* @param pVCpu The cross context virtual CPU structure.
9000	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9001	* out-of-sync. Make sure to update the required fields
9002	* before using them.
9003	* @param pVmxTransient Pointer to the VMX transient structure.
9004	*
9005	* @remarks Called with preemption disabled.
9006	* @remarks No-long-jump zone!!!
9007	*/
9008	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9009	{
9010	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9011	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9012	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9013
9014	/*
9015	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9016	*/
9017	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9018	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9019
9020	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9021	if (!CPUMIsGuestFPUStateActive(pVCpu))
9022	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9023	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9024	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9025	#endif
9026
9027	if ( pVCpu->hm.s.fPreloadGuestFpu
9028	&& !CPUMIsGuestFPUStateActive(pVCpu))
9029	{
9030	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9031	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9032	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9033	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9034	}
9035
9036	/*
9037	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9038	*/
9039	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9040	&& pVCpu->hm.s.vmx.cMsrs > 0)
9041	{
9042	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9043	}
9044
9045	/*
9046	* Load the host state bits as we may've been preempted (only happens when
9047	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9048	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9049	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9050	* See @bugref{8432}.
9051	*/
9052	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9053	{
9054	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9055	AssertRC(rc);
9056	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9057	}
9058	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9059
9060	/*
9061	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9062	*/
9063	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9064	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9065	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9066
9067	/* Store status of the shared guest-host state at the time of VM-entry. */
9068	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9069	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9070	{
9071	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9072	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9073	}
9074	else
9075	#endif
9076	{
9077	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9078	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9079	}
9080	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9081
9082	/*
9083	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9084	*/
9085	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9086	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9087
9088	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9089	RTCPUID idCurrentCpu = pCpu->idCpu;
9090	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9091	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9092	{
9093	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9094	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9095	}
9096
9097	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9098	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9099	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9100	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9101
9102	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9103
9104	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9105	to start executing. */
9106
9107	/*
9108	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9109	*/
9110	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9111	{
9112	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9113	{
9114	bool fMsrUpdated;
9115	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9116	AssertRC(rc2);
9117	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9118
9119	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9120	&fMsrUpdated);
9121	AssertRC(rc2);
9122	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9123
9124	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9125	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9126	}
9127	else
9128	{
9129	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9130	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9131	}
9132	}
9133
9134	#ifdef VBOX_STRICT
9135	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9136	hmR0VmxCheckHostEferMsr(pVCpu);
9137	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9138	#endif
9139	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9140	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9141	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9142	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9143	#endif
9144	}
9145
9146
9147	/**
9148	* Performs some essential restoration of state after running guest code in
9149	* VT-x.
9150	*
9151	* @param pVM The cross context VM structure.
9152	* @param pVCpu The cross context virtual CPU structure.
9153	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9154	* out-of-sync. Make sure to update the required fields
9155	* before using them.
9156	* @param pVmxTransient Pointer to the VMX transient structure.
9157	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9158	*
9159	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9160	*
9161	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9162	* unconditionally when it is safe to do so.
9163	*/
9164	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9165	{
9166	NOREF(pVM);
9167
9168	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9169
9170	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9171	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9172	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9173	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9174	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9175	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9176
9177	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9178	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9179
9180	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9181	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9182	Assert(!ASMIntAreEnabled());
9183	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9184
9185	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9186	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9187	{
9188	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9189	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9190	}
9191	#endif
9192
9193	#if HC_ARCH_BITS == 64
9194	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9195	#endif
9196	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9197	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9198	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9199	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9200	#else
9201	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9202	#endif
9203	#ifdef VBOX_STRICT
9204	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9205	#endif
9206	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9207	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
9208
9209	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9210	uint32_t uExitReason;
9211	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9212	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9213	AssertRC(rc);
9214	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9215	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9216
9217	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
9218	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
9219	{
9220	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9221	pVmxTransient->fVMEntryFailed));
9222	return;
9223	}
9224
9225	/*
9226	* Update the VM-exit history array here even if the VM-entry failed due to:
9227	* - Invalid guest state.
9228	* - MSR loading.
9229	* - Machine-check event.
9230	*
9231	* In any of the above cases we will still have a "valid" VM-exit reason
9232	* despite @a fVMEntryFailed being false.
9233	*
9234	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9235	*/
9236	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9237
9238	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
9239	{
9240	/** @todo We can optimize this by only syncing with our force-flags when
9241	* really needed and keeping the VMCS state as it is for most
9242	* VM-exits. */
9243	/* Update the guest interruptibility-state from the VMCS. */
9244	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9245
9246	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9247	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9248	AssertRC(rc);
9249	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9250	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9251	AssertRC(rc);
9252	#endif
9253
9254	/*
9255	* Sync the TPR shadow with our APIC state.
9256	*/
9257	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9258	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9259	{
9260	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9261	AssertRC(rc);
9262	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9263	}
9264	}
9265	}
9266
9267
9268	/**
9269	* Runs the guest code using VT-x the normal way.
9270	*
9271	* @returns VBox status code.
9272	* @param pVM The cross context VM structure.
9273	* @param pVCpu The cross context virtual CPU structure.
9274	* @param pCtx Pointer to the guest-CPU context.
9275	*
9276	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9277	*/
9278	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9279	{
9280	VMXTRANSIENT VmxTransient;
9281	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9282	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9283	uint32_t cLoops = 0;
9284
9285	for (;; cLoops++)
9286	{
9287	Assert(!HMR0SuspendPending());
9288	HMVMX_ASSERT_CPU_SAFE();
9289
9290	/* Preparatory work for running guest code, this may force us to return
9291	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9292	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9293	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9294	if (rcStrict != VINF_SUCCESS)
9295	break;
9296
9297	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9298	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9299	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9300
9301	/* Restore any residual host-state and save any bits shared between host
9302	and guest into the guest-CPU state. Re-enables interrupts! */
9303	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9304
9305	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9306	if (RT_SUCCESS(rcRun))
9307	{ /* very likely */ }
9308	else
9309	{
9310	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9311	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9312	return rcRun;
9313	}
9314
9315	/* Profile the VM-exit. */
9316	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9318	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9319	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9320	HMVMX_START_EXIT_DISPATCH_PROF();
9321
9322	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9323
9324	/* Handle the VM-exit. */
9325	#ifdef HMVMX_USE_FUNCTION_TABLE
9326	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9327	#else
9328	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9329	#endif
9330	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9331	if (rcStrict == VINF_SUCCESS)
9332	{
9333	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9334	continue; /* likely */
9335	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9336	rcStrict = VINF_EM_RAW_INTERRUPT;
9337	}
9338	break;
9339	}
9340
9341	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9342	return rcStrict;
9343	}
9344
9345
9346
9347	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9348	* probes.
9349	*
9350	* The following few functions and associated structure contains the bloat
9351	* necessary for providing detailed debug events and dtrace probes as well as
9352	* reliable host side single stepping. This works on the principle of
9353	* "subclassing" the normal execution loop and workers. We replace the loop
9354	* method completely and override selected helpers to add necessary adjustments
9355	* to their core operation.
9356	*
9357	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9358	* any performance for debug and analysis features.
9359	*
9360	* @{
9361	*/
9362
9363	/**
9364	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9365	* the debug run loop.
9366	*/
9367	typedef struct VMXRUNDBGSTATE
9368	{
9369	/** The RIP we started executing at. This is for detecting that we stepped. */
9370	uint64_t uRipStart;
9371	/** The CS we started executing with. */
9372	uint16_t uCsStart;
9373
9374	/** Whether we've actually modified the 1st execution control field. */
9375	bool fModifiedProcCtls : 1;
9376	/** Whether we've actually modified the 2nd execution control field. */
9377	bool fModifiedProcCtls2 : 1;
9378	/** Whether we've actually modified the exception bitmap. */
9379	bool fModifiedXcptBitmap : 1;
9380
9381	/** We desire the modified the CR0 mask to be cleared. */
9382	bool fClearCr0Mask : 1;
9383	/** We desire the modified the CR4 mask to be cleared. */
9384	bool fClearCr4Mask : 1;
9385	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9386	uint32_t fCpe1Extra;
9387	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9388	uint32_t fCpe1Unwanted;
9389	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9390	uint32_t fCpe2Extra;
9391	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9392	uint32_t bmXcptExtra;
9393	/** The sequence number of the Dtrace provider settings the state was
9394	* configured against. */
9395	uint32_t uDtraceSettingsSeqNo;
9396	/** VM-exits to check (one bit per VM-exit). */
9397	uint32_t bmExitsToCheck[3];
9398
9399	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9400	uint32_t fProcCtlsInitial;
9401	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9402	uint32_t fProcCtls2Initial;
9403	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9404	uint32_t bmXcptInitial;
9405	} VMXRUNDBGSTATE;
9406	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9407	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9408
9409
9410	/**
9411	* Initializes the VMXRUNDBGSTATE structure.
9412	*
9413	* @param pVCpu The cross context virtual CPU structure of the
9414	* calling EMT.
9415	* @param pCtx The CPU register context to go with @a pVCpu.
9416	* @param pDbgState The structure to initialize.
9417	*/
9418	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9419	{
9420	pDbgState->uRipStart = pCtx->rip;
9421	pDbgState->uCsStart = pCtx->cs.Sel;
9422
9423	pDbgState->fModifiedProcCtls = false;
9424	pDbgState->fModifiedProcCtls2 = false;
9425	pDbgState->fModifiedXcptBitmap = false;
9426	pDbgState->fClearCr0Mask = false;
9427	pDbgState->fClearCr4Mask = false;
9428	pDbgState->fCpe1Extra = 0;
9429	pDbgState->fCpe1Unwanted = 0;
9430	pDbgState->fCpe2Extra = 0;
9431	pDbgState->bmXcptExtra = 0;
9432	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9433	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9434	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9435	}
9436
9437
9438	/**
9439	* Updates the VMSC fields with changes requested by @a pDbgState.
9440	*
9441	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9442	* immediately before executing guest code, i.e. when interrupts are disabled.
9443	* We don't check status codes here as we cannot easily assert or return in the
9444	* latter case.
9445	*
9446	* @param pVCpu The cross context virtual CPU structure.
9447	* @param pDbgState The debug state.
9448	*/
9449	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9450	{
9451	/*
9452	* Ensure desired flags in VMCS control fields are set.
9453	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9454	*
9455	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9456	* there should be no stale data in pCtx at this point.
9457	*/
9458	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9459	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9460	{
9461	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9462	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9463	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9464	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9465	pDbgState->fModifiedProcCtls = true;
9466	}
9467
9468	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9469	{
9470	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9471	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9472	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9473	pDbgState->fModifiedProcCtls2 = true;
9474	}
9475
9476	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9477	{
9478	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9479	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9480	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9481	pDbgState->fModifiedXcptBitmap = true;
9482	}
9483
9484	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9485	{
9486	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9487	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9488	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9489	}
9490
9491	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9492	{
9493	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9494	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9495	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9496	}
9497	}
9498
9499
9500	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9501	{
9502	/*
9503	* Restore VM-exit control settings as we may not reenter this function the
9504	* next time around.
9505	*/
9506	/* We reload the initial value, trigger what we can of recalculations the
9507	next time around. From the looks of things, that's all that's required atm. */
9508	if (pDbgState->fModifiedProcCtls)
9509	{
9510	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9511	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9512	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9513	AssertRCReturn(rc2, rc2);
9514	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9515	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_DEBUG);
9516	}
9517
9518	/* We're currently the only ones messing with this one, so just restore the
9519	cached value and reload the field. */
9520	if ( pDbgState->fModifiedProcCtls2
9521	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9522	{
9523	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9524	AssertRCReturn(rc2, rc2);
9525	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9526	}
9527
9528	/* If we've modified the exception bitmap, we restore it and trigger
9529	reloading and partial recalculation the next time around. */
9530	if (pDbgState->fModifiedXcptBitmap)
9531	{
9532	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9533	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS \| HM_CHANGED_GUEST_CR0);
9534	}
9535
9536	/* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9537	if (pDbgState->fClearCr0Mask)
9538	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9539
9540	/* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9541	if (pDbgState->fClearCr4Mask)
9542	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9543
9544	return rcStrict;
9545	}
9546
9547
9548	/**
9549	* Configures VM-exit controls for current DBGF and DTrace settings.
9550	*
9551	* This updates @a pDbgState and the VMCS execution control fields to reflect
9552	* the necessary VM-exits demanded by DBGF and DTrace.
9553	*
9554	* @param pVM The cross context VM structure.
9555	* @param pVCpu The cross context virtual CPU structure.
9556	* @param pCtx Pointer to the guest-CPU context.
9557	* @param pDbgState The debug state.
9558	* @param pVmxTransient Pointer to the VMX transient structure. May update
9559	* fUpdateTscOffsettingAndPreemptTimer.
9560	*/
9561	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9562	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9563	{
9564	/*
9565	* Take down the dtrace serial number so we can spot changes.
9566	*/
9567	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9568	ASMCompilerBarrier();
9569
9570	/*
9571	* We'll rebuild most of the middle block of data members (holding the
9572	* current settings) as we go along here, so start by clearing it all.
9573	*/
9574	pDbgState->bmXcptExtra = 0;
9575	pDbgState->fCpe1Extra = 0;
9576	pDbgState->fCpe1Unwanted = 0;
9577	pDbgState->fCpe2Extra = 0;
9578	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9579	pDbgState->bmExitsToCheck[i] = 0;
9580
9581	/*
9582	* Software interrupts (INT XXh) - no idea how to trigger these...
9583	*/
9584	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9585	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9586	{
9587	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9588	}
9589
9590	/*
9591	* INT3 breakpoints - triggered by #BP exceptions.
9592	*/
9593	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9594	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9595
9596	/*
9597	* Exception bitmap and XCPT events+probes.
9598	*/
9599	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9600	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9601	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9602
9603	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9604	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9605	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9606	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9607	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9608	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9609	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9610	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9611	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9612	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9613	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9614	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9615	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9616	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9617	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9618	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9619	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9620	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9621
9622	if (pDbgState->bmXcptExtra)
9623	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9624
9625	/*
9626	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9627	*
9628	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9629	* So, when adding/changing/removing please don't forget to update it.
9630	*
9631	* Some of the macros are picking up local variables to save horizontal space,
9632	* (being able to see it in a table is the lesser evil here).
9633	*/
9634	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9635	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9636	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9637	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9638	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9639	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9640	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9641	} else do { } while (0)
9642	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9643	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9644	{ \
9645	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9646	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9647	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9648	} else do { } while (0)
9649	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9650	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9651	{ \
9652	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9653	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9654	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9655	} else do { } while (0)
9656	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9657	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9658	{ \
9659	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9660	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9661	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9662	} else do { } while (0)
9663
9664	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9665	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9666	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9667	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9668	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9669
9670	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9671	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9672	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9673	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9674	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9675	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9676	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9677	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9678	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9679	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9680	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9681	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9682	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9683	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9684	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9685	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9686	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9687	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9688	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9689	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9690	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9691	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9692	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9693	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9694	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9695	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9696	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9697	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9698	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9699	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9700	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9701	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9702	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9703	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9704	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9705	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9706
9707	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9708	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9709	{
9710	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9711	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9712	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9713	AssertRC(rc2);
9714
9715	#if 0 /** @todo fix me */
9716	pDbgState->fClearCr0Mask = true;
9717	pDbgState->fClearCr4Mask = true;
9718	#endif
9719	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9720	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9721	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9722	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9723	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9724	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9725	require clearing here and in the loop if we start using it. */
9726	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9727	}
9728	else
9729	{
9730	if (pDbgState->fClearCr0Mask)
9731	{
9732	pDbgState->fClearCr0Mask = false;
9733	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9734	}
9735	if (pDbgState->fClearCr4Mask)
9736	{
9737	pDbgState->fClearCr4Mask = false;
9738	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9739	}
9740	}
9741	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9742	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9743
9744	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9745	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9746	{
9747	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9748	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9749	}
9750	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9751	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9752
9753	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9754	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9755	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9756	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9757	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9758	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9759	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9761	#if 0 /** @todo too slow, fix handler. */
9762	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9763	#endif
9764	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9765
9766	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9767	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9768	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9769	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9770	{
9771	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9772	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9773	}
9774	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9775	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9776	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9777	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9778
9779	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9780	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9781	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9782	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9783	{
9784	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9785	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9786	}
9787	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9788	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9789	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9790	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9791
9792	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9793	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9794	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9795	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9796	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9797	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9798	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9799	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9800	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9801	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9802	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9803	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9804	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9806	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9807	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9808	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9809	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9810	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9811	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9812	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9813	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9814
9815	#undef IS_EITHER_ENABLED
9816	#undef SET_ONLY_XBM_IF_EITHER_EN
9817	#undef SET_CPE1_XBM_IF_EITHER_EN
9818	#undef SET_CPEU_XBM_IF_EITHER_EN
9819	#undef SET_CPE2_XBM_IF_EITHER_EN
9820
9821	/*
9822	* Sanitize the control stuff.
9823	*/
9824	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9825	if (pDbgState->fCpe2Extra)
9826	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9827	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9828	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9829	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9830	{
9831	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9832	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9833	}
9834
9835	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9836	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9837	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9838	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9839	}
9840
9841
9842	/**
9843	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9844	* appropriate.
9845	*
9846	* The caller has checked the VM-exit against the
9847	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9848	* already, so we don't have to do that either.
9849	*
9850	* @returns Strict VBox status code (i.e. informational status codes too).
9851	* @param pVM The cross context VM structure.
9852	* @param pVCpu The cross context virtual CPU structure.
9853	* @param pMixedCtx Pointer to the guest-CPU context.
9854	* @param pVmxTransient Pointer to the VMX-transient structure.
9855	* @param uExitReason The VM-exit reason.
9856	*
9857	* @remarks The name of this function is displayed by dtrace, so keep it short
9858	* and to the point. No longer than 33 chars long, please.
9859	*/
9860	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9861	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9862	{
9863	/*
9864	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9865	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9866	*
9867	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9868	* does. Must add/change/remove both places. Same ordering, please.
9869	*
9870	* Added/removed events must also be reflected in the next section
9871	* where we dispatch dtrace events.
9872	*/
9873	bool fDtrace1 = false;
9874	bool fDtrace2 = false;
9875	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9876	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9877	uint32_t uEventArg = 0;
9878	#define SET_EXIT(a_EventSubName) \
9879	do { \
9880	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9881	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9882	} while (0)
9883	#define SET_BOTH(a_EventSubName) \
9884	do { \
9885	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9886	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9887	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9888	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9889	} while (0)
9890	switch (uExitReason)
9891	{
9892	case VMX_EXIT_MTF:
9893	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9894
9895	case VMX_EXIT_XCPT_OR_NMI:
9896	{
9897	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9898	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9899	{
9900	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9901	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9902	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9903	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9904	{
9905	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9906	{
9907	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9908	uEventArg = pVmxTransient->uExitIntErrorCode;
9909	}
9910	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9911	switch (enmEvent1)
9912	{
9913	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9914	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9915	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9916	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9917	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9918	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9919	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9920	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9921	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9922	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9923	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9924	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9925	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9926	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9927	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9928	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9929	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9930	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9931	default: break;
9932	}
9933	}
9934	else
9935	AssertFailed();
9936	break;
9937
9938	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9939	uEventArg = idxVector;
9940	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9941	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9942	break;
9943	}
9944	break;
9945	}
9946
9947	case VMX_EXIT_TRIPLE_FAULT:
9948	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9949	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9950	break;
9951	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9952	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9953	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9954	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9955	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9956
9957	/* Instruction specific VM-exits: */
9958	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9959	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9960	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9961	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9962	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9963	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9964	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9965	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9966	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9967	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9968	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9969	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9970	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9971	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9972	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9973	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9974	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9975	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9976	case VMX_EXIT_MOV_CRX:
9977	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9978	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9979	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9980	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9981	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9982	SET_BOTH(CRX_READ);
9983	else
9984	SET_BOTH(CRX_WRITE);
9985	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9986	break;
9987	case VMX_EXIT_MOV_DRX:
9988	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9989	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9990	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9991	SET_BOTH(DRX_READ);
9992	else
9993	SET_BOTH(DRX_WRITE);
9994	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9995	break;
9996	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9997	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9998	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9999	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10000	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10001	case VMX_EXIT_XDTR_ACCESS:
10002	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10003	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10004	{
10005	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10006	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10007	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10008	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10009	}
10010	break;
10011
10012	case VMX_EXIT_TR_ACCESS:
10013	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10014	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10015	{
10016	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10017	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10018	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10019	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10020	}
10021	break;
10022
10023	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10024	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10025	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10026	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10027	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10028	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10029	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10030	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10031	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10032	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10033	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10034
10035	/* Events that aren't relevant at this point. */
10036	case VMX_EXIT_EXT_INT:
10037	case VMX_EXIT_INT_WINDOW:
10038	case VMX_EXIT_NMI_WINDOW:
10039	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10040	case VMX_EXIT_PREEMPT_TIMER:
10041	case VMX_EXIT_IO_INSTR:
10042	break;
10043
10044	/* Errors and unexpected events. */
10045	case VMX_EXIT_INIT_SIGNAL:
10046	case VMX_EXIT_SIPI:
10047	case VMX_EXIT_IO_SMI:
10048	case VMX_EXIT_SMI:
10049	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10050	case VMX_EXIT_ERR_MSR_LOAD:
10051	case VMX_EXIT_ERR_MACHINE_CHECK:
10052	break;
10053
10054	default:
10055	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10056	break;
10057	}
10058	#undef SET_BOTH
10059	#undef SET_EXIT
10060
10061	/*
10062	* Dtrace tracepoints go first. We do them here at once so we don't
10063	* have to copy the guest state saving and stuff a few dozen times.
10064	* Down side is that we've got to repeat the switch, though this time
10065	* we use enmEvent since the probes are a subset of what DBGF does.
10066	*/
10067	if (fDtrace1 \|\| fDtrace2)
10068	{
10069	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10070	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10071	switch (enmEvent1)
10072	{
10073	/** @todo consider which extra parameters would be helpful for each probe. */
10074	case DBGFEVENT_END: break;
10075	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10076	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10077	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10078	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10079	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10080	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10081	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10082	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10083	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10084	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10085	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10086	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10087	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10088	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10089	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10090	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10091	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10092	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10093	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10094	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10095	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10096	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10097	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10098	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10099	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10100	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10101	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10102	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10103	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10104	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10105	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10106	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10107	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10108	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10109	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10110	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10111	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10112	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10113	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10114	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10115	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10116	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10117	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10118	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10119	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10120	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10121	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10122	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10123	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10124	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10125	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10126	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10127	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10128	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10129	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10130	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10131	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10132	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10133	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10134	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10135	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10136	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10137	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10138	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10139	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10140	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10141	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10142	}
10143	switch (enmEvent2)
10144	{
10145	/** @todo consider which extra parameters would be helpful for each probe. */
10146	case DBGFEVENT_END: break;
10147	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10148	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10149	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10150	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10151	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10152	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10153	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10154	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10155	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10156	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10157	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10158	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10159	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10160	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10161	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10162	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10163	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10164	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10165	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10166	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10167	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10173	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10174	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10175	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10176	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10177	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10178	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10179	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10180	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10181	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10182	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10183	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10184	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10185	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10186	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10187	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10188	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10189	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10190	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10191	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10192	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10193	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10194	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10195	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10196	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10197	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10198	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10199	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10200	}
10201	}
10202
10203	/*
10204	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10205	* the DBGF call will do a full check).
10206	*
10207	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10208	* Note! If we have to events, we prioritize the first, i.e. the instruction
10209	* one, in order to avoid event nesting.
10210	*/
10211	if ( enmEvent1 != DBGFEVENT_END
10212	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10213	{
10214	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10215	if (rcStrict != VINF_SUCCESS)
10216	return rcStrict;
10217	}
10218	else if ( enmEvent2 != DBGFEVENT_END
10219	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10220	{
10221	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10222	if (rcStrict != VINF_SUCCESS)
10223	return rcStrict;
10224	}
10225
10226	return VINF_SUCCESS;
10227	}
10228
10229
10230	/**
10231	* Single-stepping VM-exit filtering.
10232	*
10233	* This is preprocessing the VM-exits and deciding whether we've gotten far
10234	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10235	* handling is performed.
10236	*
10237	* @returns Strict VBox status code (i.e. informational status codes too).
10238	* @param pVM The cross context VM structure.
10239	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10240	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10241	* out-of-sync. Make sure to update the required
10242	* fields before using them.
10243	* @param pVmxTransient Pointer to the VMX-transient structure.
10244	* @param uExitReason The VM-exit reason.
10245	* @param pDbgState The debug state.
10246	*/
10247	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10248	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10249	{
10250	/*
10251	* Expensive (saves context) generic dtrace VM-exit probe.
10252	*/
10253	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10254	{ /* more likely */ }
10255	else
10256	{
10257	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10258	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10259	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10260	}
10261
10262	/*
10263	* Check for host NMI, just to get that out of the way.
10264	*/
10265	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10266	{ /* normally likely */ }
10267	else
10268	{
10269	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10270	AssertRCReturn(rc2, rc2);
10271	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10272	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10273	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10274	}
10275
10276	/*
10277	* Check for single stepping event if we're stepping.
10278	*/
10279	if (pVCpu->hm.s.fSingleInstruction)
10280	{
10281	switch (uExitReason)
10282	{
10283	case VMX_EXIT_MTF:
10284	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10285
10286	/* Various events: */
10287	case VMX_EXIT_XCPT_OR_NMI:
10288	case VMX_EXIT_EXT_INT:
10289	case VMX_EXIT_TRIPLE_FAULT:
10290	case VMX_EXIT_INT_WINDOW:
10291	case VMX_EXIT_NMI_WINDOW:
10292	case VMX_EXIT_TASK_SWITCH:
10293	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10294	case VMX_EXIT_APIC_ACCESS:
10295	case VMX_EXIT_EPT_VIOLATION:
10296	case VMX_EXIT_EPT_MISCONFIG:
10297	case VMX_EXIT_PREEMPT_TIMER:
10298
10299	/* Instruction specific VM-exits: */
10300	case VMX_EXIT_CPUID:
10301	case VMX_EXIT_GETSEC:
10302	case VMX_EXIT_HLT:
10303	case VMX_EXIT_INVD:
10304	case VMX_EXIT_INVLPG:
10305	case VMX_EXIT_RDPMC:
10306	case VMX_EXIT_RDTSC:
10307	case VMX_EXIT_RSM:
10308	case VMX_EXIT_VMCALL:
10309	case VMX_EXIT_VMCLEAR:
10310	case VMX_EXIT_VMLAUNCH:
10311	case VMX_EXIT_VMPTRLD:
10312	case VMX_EXIT_VMPTRST:
10313	case VMX_EXIT_VMREAD:
10314	case VMX_EXIT_VMRESUME:
10315	case VMX_EXIT_VMWRITE:
10316	case VMX_EXIT_VMXOFF:
10317	case VMX_EXIT_VMXON:
10318	case VMX_EXIT_MOV_CRX:
10319	case VMX_EXIT_MOV_DRX:
10320	case VMX_EXIT_IO_INSTR:
10321	case VMX_EXIT_RDMSR:
10322	case VMX_EXIT_WRMSR:
10323	case VMX_EXIT_MWAIT:
10324	case VMX_EXIT_MONITOR:
10325	case VMX_EXIT_PAUSE:
10326	case VMX_EXIT_XDTR_ACCESS:
10327	case VMX_EXIT_TR_ACCESS:
10328	case VMX_EXIT_INVEPT:
10329	case VMX_EXIT_RDTSCP:
10330	case VMX_EXIT_INVVPID:
10331	case VMX_EXIT_WBINVD:
10332	case VMX_EXIT_XSETBV:
10333	case VMX_EXIT_RDRAND:
10334	case VMX_EXIT_INVPCID:
10335	case VMX_EXIT_VMFUNC:
10336	case VMX_EXIT_RDSEED:
10337	case VMX_EXIT_XSAVES:
10338	case VMX_EXIT_XRSTORS:
10339	{
10340	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10341	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10342	AssertRCReturn(rc2, rc2);
10343	if ( pMixedCtx->rip != pDbgState->uRipStart
10344	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10345	return VINF_EM_DBG_STEPPED;
10346	break;
10347	}
10348
10349	/* Errors and unexpected events: */
10350	case VMX_EXIT_INIT_SIGNAL:
10351	case VMX_EXIT_SIPI:
10352	case VMX_EXIT_IO_SMI:
10353	case VMX_EXIT_SMI:
10354	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10355	case VMX_EXIT_ERR_MSR_LOAD:
10356	case VMX_EXIT_ERR_MACHINE_CHECK:
10357	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10358	break;
10359
10360	default:
10361	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10362	break;
10363	}
10364	}
10365
10366	/*
10367	* Check for debugger event breakpoints and dtrace probes.
10368	*/
10369	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10370	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10371	{
10372	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10373	if (rcStrict != VINF_SUCCESS)
10374	return rcStrict;
10375	}
10376
10377	/*
10378	* Normal processing.
10379	*/
10380	#ifdef HMVMX_USE_FUNCTION_TABLE
10381	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10382	#else
10383	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10384	#endif
10385	}
10386
10387
10388	/**
10389	* Single steps guest code using VT-x.
10390	*
10391	* @returns Strict VBox status code (i.e. informational status codes too).
10392	* @param pVM The cross context VM structure.
10393	* @param pVCpu The cross context virtual CPU structure.
10394	* @param pCtx Pointer to the guest-CPU context.
10395	*
10396	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10397	*/
10398	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10399	{
10400	VMXTRANSIENT VmxTransient;
10401	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10402
10403	/* Set HMCPU indicators. */
10404	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10405	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10406	pVCpu->hm.s.fDebugWantRdTscExit = false;
10407	pVCpu->hm.s.fUsingDebugLoop = true;
10408
10409	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10410	VMXRUNDBGSTATE DbgState;
10411	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10412	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10413
10414	/*
10415	* The loop.
10416	*/
10417	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10418	for (uint32_t cLoops = 0; ; cLoops++)
10419	{
10420	Assert(!HMR0SuspendPending());
10421	HMVMX_ASSERT_CPU_SAFE();
10422	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10423
10424	/*
10425	* Preparatory work for running guest code, this may force us to return
10426	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10427	*/
10428	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10429	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10430	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10431	if (rcStrict != VINF_SUCCESS)
10432	break;
10433
10434	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10435	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10436
10437	/*
10438	* Now we can run the guest code.
10439	*/
10440	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10441
10442	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10443
10444	/*
10445	* Restore any residual host-state and save any bits shared between host
10446	* and guest into the guest-CPU state. Re-enables interrupts!
10447	*/
10448	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10449
10450	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10451	if (RT_SUCCESS(rcRun))
10452	{ /* very likely */ }
10453	else
10454	{
10455	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10456	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10457	return rcRun;
10458	}
10459
10460	/* Profile the VM-exit. */
10461	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10463	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10464	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10465	HMVMX_START_EXIT_DISPATCH_PROF();
10466
10467	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10468
10469	/*
10470	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10471	*/
10472	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10473	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10474	if (rcStrict != VINF_SUCCESS)
10475	break;
10476	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10477	{
10478	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10479	rcStrict = VINF_EM_RAW_INTERRUPT;
10480	break;
10481	}
10482
10483	/*
10484	* Stepping: Did the RIP change, if so, consider it a single step.
10485	* Otherwise, make sure one of the TFs gets set.
10486	*/
10487	if (fStepping)
10488	{
10489	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10490	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10491	AssertRCReturn(rc2, rc2);
10492	if ( pCtx->rip != DbgState.uRipStart
10493	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10494	{
10495	rcStrict = VINF_EM_DBG_STEPPED;
10496	break;
10497	}
10498	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10499	}
10500
10501	/*
10502	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10503	*/
10504	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10505	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10506	}
10507
10508	/*
10509	* Clear the X86_EFL_TF if necessary.
10510	*/
10511	if (pVCpu->hm.s.fClearTrapFlag)
10512	{
10513	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10514	AssertRCReturn(rc2, rc2);
10515	pVCpu->hm.s.fClearTrapFlag = false;
10516	pCtx->eflags.Bits.u1TF = 0;
10517	}
10518	/** @todo there seems to be issues with the resume flag when the monitor trap
10519	* flag is pending without being used. Seen early in bios init when
10520	* accessing APIC page in protected mode. */
10521
10522	/*
10523	* Restore VM-exit control settings as we may not reenter this function the
10524	* next time around.
10525	*/
10526	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10527
10528	/* Restore HMCPU indicators. */
10529	pVCpu->hm.s.fUsingDebugLoop = false;
10530	pVCpu->hm.s.fDebugWantRdTscExit = false;
10531	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10532
10533	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10534	return rcStrict;
10535	}
10536
10537
10538	/** @} */
10539
10540
10541	/**
10542	* Checks if any expensive dtrace probes are enabled and we should go to the
10543	* debug loop.
10544	*
10545	* @returns true if we should use debug loop, false if not.
10546	*/
10547	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10548	{
10549	/* It's probably faster to OR the raw 32-bit counter variables together.
10550	Since the variables are in an array and the probes are next to one
10551	another (more or less), we have good locality. So, better read
10552	eight-nine cache lines ever time and only have one conditional, than
10553	128+ conditionals, right? */
10554	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10555	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10556	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10557	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10558	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10559	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10560	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10561	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10562	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10563	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10564	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10565	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10566	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10567	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10568	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10569	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10570	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10571	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10572	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10573	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10574	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10575	) != 0
10576	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10577	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10578	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10579	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10580	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10581	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10582	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10583	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10584	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10585	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10586	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10587	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10588	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10589	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10590	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10591	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10592	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10593	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10594	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10595	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10596	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10597	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10598	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10599	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10600	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10601	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10602	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10603	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10604	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10605	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10606	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10607	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10608	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10609	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10610	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10611	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10612	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10613	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10614	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10615	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10616	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10617	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10618	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10619	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10620	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10621	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10622	) != 0
10623	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10624	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10625	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10626	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10627	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10628	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10629	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10630	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10631	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10632	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10633	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10634	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10635	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10636	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10637	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10638	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10639	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10640	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10641	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10642	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10643	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10644	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10645	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10646	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10647	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10648	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10649	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10650	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10651	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10652	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10653	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10654	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10655	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10656	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10657	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10658	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10659	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10660	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10661	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10662	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10663	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10664	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10665	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10666	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10667	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10668	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10669	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10670	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10671	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10672	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10673	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10674	) != 0;
10675	}
10676
10677
10678	/**
10679	* Runs the guest code using VT-x.
10680	*
10681	* @returns Strict VBox status code (i.e. informational status codes too).
10682	* @param pVM The cross context VM structure.
10683	* @param pVCpu The cross context virtual CPU structure.
10684	* @param pCtx Pointer to the guest-CPU context.
10685	*/
10686	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10687	{
10688	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10689	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10690	HMVMX_ASSERT_PREEMPT_SAFE();
10691
10692	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10693
10694	VBOXSTRICTRC rcStrict;
10695	if ( !pVCpu->hm.s.fUseDebugLoop
10696	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10697	&& !DBGFIsStepping(pVCpu)
10698	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10699	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10700	else
10701	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10702
10703	if (rcStrict == VERR_EM_INTERPRETER)
10704	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10705	else if (rcStrict == VINF_EM_RESET)
10706	rcStrict = VINF_EM_TRIPLE_FAULT;
10707
10708	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10709	if (RT_FAILURE(rc2))
10710	{
10711	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10712	rcStrict = rc2;
10713	}
10714	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10715	return rcStrict;
10716	}
10717
10718
10719	#ifndef HMVMX_USE_FUNCTION_TABLE
10720	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10721	{
10722	# ifdef DEBUG_ramshankar
10723	# define RETURN_EXIT_CALL(a_CallExpr) \
10724	do { \
10725	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10726	VBOXSTRICTRC rcStrict = a_CallExpr; \
10727	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10728	return rcStrict; \
10729	} while (0)
10730	# else
10731	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10732	# endif
10733	switch (rcReason)
10734	{
10735	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10736	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10737	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10738	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10739	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10740	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10741	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10742	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10743	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10744	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10745	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10746	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10747	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10748	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10749	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10750	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10751	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10752	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10753	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10754	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10755	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10756	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10757	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10758	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10759	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10760	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10761	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10762	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10763	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10764	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10765	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10766	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10767	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10768	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10769
10770	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10771	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10772	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10773	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10774	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10775	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10776	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10777	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10778	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10779
10780	case VMX_EXIT_VMCLEAR:
10781	case VMX_EXIT_VMLAUNCH:
10782	case VMX_EXIT_VMPTRLD:
10783	case VMX_EXIT_VMPTRST:
10784	case VMX_EXIT_VMREAD:
10785	case VMX_EXIT_VMRESUME:
10786	case VMX_EXIT_VMWRITE:
10787	case VMX_EXIT_VMXOFF:
10788	case VMX_EXIT_VMXON:
10789	case VMX_EXIT_INVEPT:
10790	case VMX_EXIT_INVVPID:
10791	case VMX_EXIT_VMFUNC:
10792	case VMX_EXIT_XSAVES:
10793	case VMX_EXIT_XRSTORS:
10794	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10795	case VMX_EXIT_ENCLS:
10796	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10797	case VMX_EXIT_PML_FULL:
10798	default:
10799	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10800	}
10801	#undef RETURN_EXIT_CALL
10802	}
10803	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10804
10805
10806	#ifdef VBOX_STRICT
10807	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10808	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10809	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10810
10811	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10812	do { \
10813	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10814	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10815	} while (0)
10816
10817	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10818	do { \
10819	AssertPtr(pVCpu); \
10820	AssertPtr(pMixedCtx); \
10821	AssertPtr(pVmxTransient); \
10822	Assert(pVmxTransient->fVMEntryFailed == false); \
10823	Assert(ASMIntAreEnabled()); \
10824	HMVMX_ASSERT_PREEMPT_SAFE(); \
10825	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10826	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10827	HMVMX_ASSERT_PREEMPT_SAFE(); \
10828	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10829	HMVMX_ASSERT_PREEMPT_CPUID(); \
10830	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10831	} while (0)
10832
10833	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10834	do { \
10835	Log4Func(("\n")); \
10836	} while (0)
10837	#else /* nonstrict builds: */
10838	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10839	do { \
10840	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10841	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10842	} while (0)
10843	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10844	#endif
10845
10846
10847	/**
10848	* Advances the guest RIP by the specified number of bytes.
10849	*
10850	* @param pVCpu The cross context virtual CPU structure.
10851	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10852	* out-of-sync. Make sure to update the required fields
10853	* before using them.
10854	* @param cbInstr Number of bytes to advance the RIP by.
10855	*
10856	* @remarks No-long-jump zone!!!
10857	*/
10858	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10859	{
10860	/* Advance the RIP. */
10861	pMixedCtx->rip += cbInstr;
10862	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10863
10864	/* Update interrupt inhibition. */
10865	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10866	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10867	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10868	}
10869
10870
10871	/**
10872	* Advances the guest RIP after reading it from the VMCS.
10873	*
10874	* @returns VBox status code, no informational status codes.
10875	* @param pVCpu The cross context virtual CPU structure.
10876	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10877	* out-of-sync. Make sure to update the required fields
10878	* before using them.
10879	* @param pVmxTransient Pointer to the VMX transient structure.
10880	*
10881	* @remarks No-long-jump zone!!!
10882	*/
10883	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10884	{
10885	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10886	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10887	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10888	AssertRCReturn(rc, rc);
10889
10890	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10891
10892	/*
10893	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10894	* pending debug exception field as it takes care of priority of events.
10895	*
10896	* See Intel spec. 32.2.1 "Debug Exceptions".
10897	*/
10898	if ( !pVCpu->hm.s.fSingleInstruction
10899	&& pMixedCtx->eflags.Bits.u1TF)
10900	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10901
10902	return VINF_SUCCESS;
10903	}
10904
10905
10906	/**
10907	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10908	* and update error record fields accordingly.
10909	*
10910	* @return VMX_IGS_* return codes.
10911	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10912	* wrong with the guest state.
10913	*
10914	* @param pVM The cross context VM structure.
10915	* @param pVCpu The cross context virtual CPU structure.
10916	* @param pCtx Pointer to the guest-CPU state.
10917	*
10918	* @remarks This function assumes our cache of the VMCS controls
10919	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10920	*/
10921	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10922	{
10923	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10924	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10925	uError = (err); \
10926	break; \
10927	} else do { } while (0)
10928
10929	int rc;
10930	uint32_t uError = VMX_IGS_ERROR;
10931	uint32_t u32Val;
10932	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10933
10934	do
10935	{
10936	/*
10937	* CR0.
10938	*/
10939	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10940	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10941	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10942	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10943	if (fUnrestrictedGuest)
10944	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10945
10946	uint32_t u32GuestCR0;
10947	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10948	AssertRCBreak(rc);
10949	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10950	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10951	if ( !fUnrestrictedGuest
10952	&& (u32GuestCR0 & X86_CR0_PG)
10953	&& !(u32GuestCR0 & X86_CR0_PE))
10954	{
10955	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10956	}
10957
10958	/*
10959	* CR4.
10960	*/
10961	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10962	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10963
10964	uint32_t u32GuestCR4;
10965	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10966	AssertRCBreak(rc);
10967	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10968	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10969
10970	/*
10971	* IA32_DEBUGCTL MSR.
10972	*/
10973	uint64_t u64Val;
10974	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10975	AssertRCBreak(rc);
10976	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10977	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10978	{
10979	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10980	}
10981	uint64_t u64DebugCtlMsr = u64Val;
10982
10983	#ifdef VBOX_STRICT
10984	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10985	AssertRCBreak(rc);
10986	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10987	#endif
10988	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10989
10990	/*
10991	* RIP and RFLAGS.
10992	*/
10993	uint32_t u32Eflags;
10994	#if HC_ARCH_BITS == 64
10995	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10996	AssertRCBreak(rc);
10997	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10998	if ( !fLongModeGuest
10999	\|\| !pCtx->cs.Attr.n.u1Long)
11000	{
11001	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11002	}
11003	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11004	* must be identical if the "IA-32e mode guest" VM-entry
11005	* control is 1 and CS.L is 1. No check applies if the
11006	* CPU supports 64 linear-address bits. */
11007
11008	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11009	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11010	AssertRCBreak(rc);
11011	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11012	VMX_IGS_RFLAGS_RESERVED);
11013	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11014	u32Eflags = u64Val;
11015	#else
11016	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11017	AssertRCBreak(rc);
11018	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11019	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11020	#endif
11021
11022	if ( fLongModeGuest
11023	\|\| ( fUnrestrictedGuest
11024	&& !(u32GuestCR0 & X86_CR0_PE)))
11025	{
11026	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11027	}
11028
11029	uint32_t u32EntryInfo;
11030	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11031	AssertRCBreak(rc);
11032	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11033	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11034	{
11035	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11036	}
11037
11038	/*
11039	* 64-bit checks.
11040	*/
11041	#if HC_ARCH_BITS == 64
11042	if (fLongModeGuest)
11043	{
11044	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11045	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11046	}
11047
11048	if ( !fLongModeGuest
11049	&& (u32GuestCR4 & X86_CR4_PCIDE))
11050	{
11051	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11052	}
11053
11054	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11055	* 51:32 beyond the processor's physical-address width are 0. */
11056
11057	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11058	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11059	{
11060	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11061	}
11062
11063	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11064	AssertRCBreak(rc);
11065	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11066
11067	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11068	AssertRCBreak(rc);
11069	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11070	#endif
11071
11072	/*
11073	* PERF_GLOBAL MSR.
11074	*/
11075	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11076	{
11077	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11078	AssertRCBreak(rc);
11079	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11080	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11081	}
11082
11083	/*
11084	* PAT MSR.
11085	*/
11086	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11087	{
11088	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11089	AssertRCBreak(rc);
11090	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11091	for (unsigned i = 0; i < 8; i++)
11092	{
11093	uint8_t u8Val = (u64Val & 0xff);
11094	if ( u8Val != 0 /* UC */
11095	&& u8Val != 1 /* WC */
11096	&& u8Val != 4 /* WT */
11097	&& u8Val != 5 /* WP */
11098	&& u8Val != 6 /* WB */
11099	&& u8Val != 7 /* UC- */)
11100	{
11101	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11102	}
11103	u64Val >>= 8;
11104	}
11105	}
11106
11107	/*
11108	* EFER MSR.
11109	*/
11110	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11111	{
11112	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11113	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11114	AssertRCBreak(rc);
11115	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11116	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11117	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11118	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11119	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11120	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11121	\|\| !(u32GuestCR0 & X86_CR0_PG)
11122	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11123	VMX_IGS_EFER_LMA_LME_MISMATCH);
11124	}
11125
11126	/*
11127	* Segment registers.
11128	*/
11129	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11130	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11131	if (!(u32Eflags & X86_EFL_VM))
11132	{
11133	/* CS */
11134	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11135	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11136	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11137	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11138	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11139	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11140	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11141	/* CS cannot be loaded with NULL in protected mode. */
11142	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11143	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11144	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11145	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11146	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11147	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11148	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11149	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11150	else
11151	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11152
11153	/* SS */
11154	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11155	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11156	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11157	if ( !(pCtx->cr0 & X86_CR0_PE)
11158	\|\| pCtx->cs.Attr.n.u4Type == 3)
11159	{
11160	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11161	}
11162	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11163	{
11164	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11165	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11166	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11167	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11168	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11169	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11170	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11171	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11172	}
11173
11174	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11175	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11176	{
11177	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11178	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11179	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11180	\|\| pCtx->ds.Attr.n.u4Type > 11
11181	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11182	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11183	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11184	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11185	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11186	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11187	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11188	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11189	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11190	}
11191	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11192	{
11193	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11194	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11195	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11196	\|\| pCtx->es.Attr.n.u4Type > 11
11197	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11198	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11199	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11200	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11201	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11202	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11203	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11204	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11205	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11206	}
11207	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11208	{
11209	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11210	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11211	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11212	\|\| pCtx->fs.Attr.n.u4Type > 11
11213	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11214	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11215	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11216	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11217	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11218	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11219	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11220	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11221	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11222	}
11223	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11224	{
11225	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11226	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11227	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11228	\|\| pCtx->gs.Attr.n.u4Type > 11
11229	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11230	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11231	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11232	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11233	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11234	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11235	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11236	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11237	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11238	}
11239	/* 64-bit capable CPUs. */
11240	#if HC_ARCH_BITS == 64
11241	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11242	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11243	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11244	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11245	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11246	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11247	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11248	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11249	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11250	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11251	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11252	#endif
11253	}
11254	else
11255	{
11256	/* V86 mode checks. */
11257	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11258	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11259	{
11260	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11261	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11262	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11263	}
11264	else
11265	{
11266	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11267	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11268	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11269	}
11270
11271	/* CS */
11272	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11273	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11274	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11275	/* SS */
11276	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11277	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11278	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11279	/* DS */
11280	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11281	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11282	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11283	/* ES */
11284	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11285	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11286	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11287	/* FS */
11288	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11289	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11290	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11291	/* GS */
11292	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11293	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11294	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11295	/* 64-bit capable CPUs. */
11296	#if HC_ARCH_BITS == 64
11297	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11298	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11299	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11300	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11301	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11302	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11303	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11304	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11305	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11306	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11307	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11308	#endif
11309	}
11310
11311	/*
11312	* TR.
11313	*/
11314	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11315	/* 64-bit capable CPUs. */
11316	#if HC_ARCH_BITS == 64
11317	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11318	#endif
11319	if (fLongModeGuest)
11320	{
11321	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11322	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11323	}
11324	else
11325	{
11326	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11327	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11328	VMX_IGS_TR_ATTR_TYPE_INVALID);
11329	}
11330	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11331	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11332	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11333	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11334	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11335	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11336	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11337	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11338
11339	/*
11340	* GDTR and IDTR.
11341	*/
11342	#if HC_ARCH_BITS == 64
11343	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11344	AssertRCBreak(rc);
11345	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11346
11347	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11348	AssertRCBreak(rc);
11349	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11350	#endif
11351
11352	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11353	AssertRCBreak(rc);
11354	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11355
11356	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11357	AssertRCBreak(rc);
11358	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11359
11360	/*
11361	* Guest Non-Register State.
11362	*/
11363	/* Activity State. */
11364	uint32_t u32ActivityState;
11365	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11366	AssertRCBreak(rc);
11367	HMVMX_CHECK_BREAK( !u32ActivityState
11368	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11369	VMX_IGS_ACTIVITY_STATE_INVALID);
11370	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11371	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11372	uint32_t u32IntrState;
11373	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11374	AssertRCBreak(rc);
11375	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11376	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11377	{
11378	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11379	}
11380
11381	/** @todo Activity state and injecting interrupts. Left as a todo since we
11382	* currently don't use activity states but ACTIVE. */
11383
11384	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11385	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11386
11387	/* Guest interruptibility-state. */
11388	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11389	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11390	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11391	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11392	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11393	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11394	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11395	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11396	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11397	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11398	{
11399	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11400	{
11401	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11402	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11403	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11404	}
11405	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11406	{
11407	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11408	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11409	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11410	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11411	}
11412	}
11413	/** @todo Assumes the processor is not in SMM. */
11414	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11415	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11416	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11417	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11418	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11419	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11420	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11421	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11422	{
11423	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11424	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11425	}
11426
11427	/* Pending debug exceptions. */
11428	#if HC_ARCH_BITS == 64
11429	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11430	AssertRCBreak(rc);
11431	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11432	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11433	u32Val = u64Val; /* For pending debug exceptions checks below. */
11434	#else
11435	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11436	AssertRCBreak(rc);
11437	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11438	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11439	#endif
11440
11441	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11442	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11443	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11444	{
11445	if ( (u32Eflags & X86_EFL_TF)
11446	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11447	{
11448	/* Bit 14 is PendingDebug.BS. */
11449	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11450	}
11451	if ( !(u32Eflags & X86_EFL_TF)
11452	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11453	{
11454	/* Bit 14 is PendingDebug.BS. */
11455	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11456	}
11457	}
11458
11459	/* VMCS link pointer. */
11460	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11461	AssertRCBreak(rc);
11462	if (u64Val != UINT64_C(0xffffffffffffffff))
11463	{
11464	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11465	/** @todo Bits beyond the processor's physical-address width MBZ. */
11466	/** @todo 32-bit located in memory referenced by value of this field (as a
11467	* physical address) must contain the processor's VMCS revision ID. */
11468	/** @todo SMM checks. */
11469	}
11470
11471	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11472	* not using Nested Paging? */
11473	if ( pVM->hm.s.fNestedPaging
11474	&& !fLongModeGuest
11475	&& CPUMIsGuestInPAEModeEx(pCtx))
11476	{
11477	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11478	AssertRCBreak(rc);
11479	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11480
11481	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11482	AssertRCBreak(rc);
11483	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11484
11485	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11486	AssertRCBreak(rc);
11487	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11488
11489	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11490	AssertRCBreak(rc);
11491	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11492	}
11493
11494	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11495	if (uError == VMX_IGS_ERROR)
11496	uError = VMX_IGS_REASON_NOT_FOUND;
11497	} while (0);
11498
11499	pVCpu->hm.s.u32HMError = uError;
11500	return uError;
11501
11502	#undef HMVMX_ERROR_BREAK
11503	#undef HMVMX_CHECK_BREAK
11504	}
11505
11506	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11507	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11508	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11509
11510	/** @name VM-exit handlers.
11511	* @{
11512	*/
11513
11514	/**
11515	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11516	*/
11517	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11518	{
11519	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11520	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11521	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11522	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11523	return VINF_SUCCESS;
11524	return VINF_EM_RAW_INTERRUPT;
11525	}
11526
11527
11528	/**
11529	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11530	*/
11531	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11532	{
11533	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11534	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11535
11536	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11537	AssertRCReturn(rc, rc);
11538
11539	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11540	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11541	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11542	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11543
11544	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11545	{
11546	/*
11547	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11548	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11549	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11550	*
11551	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11552	*/
11553	VMXDispatchHostNmi();
11554	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11555	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11556	return VINF_SUCCESS;
11557	}
11558
11559	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11560	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11561	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11562	{ /* likely */ }
11563	else
11564	{
11565	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11566	rcStrictRc1 = VINF_SUCCESS;
11567	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11568	return rcStrictRc1;
11569	}
11570
11571	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11572	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11573	switch (uIntType)
11574	{
11575	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11576	Assert(uVector == X86_XCPT_DB);
11577	/* fall thru */
11578	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11579	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11580	/* fall thru */
11581	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11582	{
11583	/*
11584	* If there's any exception caused as a result of event injection, the resulting
11585	* secondary/final execption will be pending, we shall continue guest execution
11586	* after injecting the event. The page-fault case is complicated and we manually
11587	* handle any currently pending event in hmR0VmxExitXcptPF.
11588	*/
11589	if (!pVCpu->hm.s.Event.fPending)
11590	{ /* likely */ }
11591	else if (uVector != X86_XCPT_PF)
11592	{
11593	rc = VINF_SUCCESS;
11594	break;
11595	}
11596
11597	switch (uVector)
11598	{
11599	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11600	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11601	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11602	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11603	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11604	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11605	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11606
11607	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11608	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11609	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11610	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11611	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11612	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11613	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11614	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11615	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11616	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11617	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11618	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11619	default:
11620	{
11621	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11622	AssertRCReturn(rc, rc);
11623
11624	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11625	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11626	{
11627	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11628	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11629	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11630
11631	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11632	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11633	AssertRCReturn(rc, rc);
11634	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11635	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11636	0 /* GCPtrFaultAddress */);
11637	AssertRCReturn(rc, rc);
11638	}
11639	else
11640	{
11641	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11642	pVCpu->hm.s.u32HMError = uVector;
11643	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11644	}
11645	break;
11646	}
11647	}
11648	break;
11649	}
11650
11651	default:
11652	{
11653	pVCpu->hm.s.u32HMError = uExitIntInfo;
11654	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11655	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11656	break;
11657	}
11658	}
11659	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11660	return rc;
11661	}
11662
11663
11664	/**
11665	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11666	*/
11667	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11668	{
11669	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11670
11671	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11672	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11673
11674	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11675	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11676	return VINF_SUCCESS;
11677	}
11678
11679
11680	/**
11681	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11682	*/
11683	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11684	{
11685	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11686	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11687	{
11688	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11689	HMVMX_RETURN_UNEXPECTED_EXIT();
11690	}
11691
11692	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11693
11694	/*
11695	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11696	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11697	*/
11698	uint32_t uIntrState = 0;
11699	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11700	AssertRCReturn(rc, rc);
11701
11702	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11703	if ( fBlockSti
11704	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11705	{
11706	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11707	}
11708
11709	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11710	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11711
11712	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11713	return VINF_SUCCESS;
11714	}
11715
11716
11717	/**
11718	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11719	*/
11720	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11721	{
11722	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11724	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11725	}
11726
11727
11728	/**
11729	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11730	*/
11731	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11732	{
11733	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11734	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11735	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11736	}
11737
11738
11739	/**
11740	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11741	*/
11742	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11743	{
11744	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11745	PVM pVM = pVCpu->CTX_SUFF(pVM);
11746	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11747	if (RT_LIKELY(rc == VINF_SUCCESS))
11748	{
11749	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11750	Assert(pVmxTransient->cbInstr == 2);
11751	}
11752	else
11753	{
11754	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11755	rc = VERR_EM_INTERPRETER;
11756	}
11757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11758	return rc;
11759	}
11760
11761
11762	/**
11763	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11764	*/
11765	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11766	{
11767	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11768	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11769	AssertRCReturn(rc, rc);
11770
11771	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11772	return VINF_EM_RAW_EMULATE_INSTR;
11773
11774	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11775	HMVMX_RETURN_UNEXPECTED_EXIT();
11776	}
11777
11778
11779	/**
11780	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11781	*/
11782	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11783	{
11784	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11785	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11786	AssertRCReturn(rc, rc);
11787
11788	PVM pVM = pVCpu->CTX_SUFF(pVM);
11789	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11790	if (RT_LIKELY(rc == VINF_SUCCESS))
11791	{
11792	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11793	Assert(pVmxTransient->cbInstr == 2);
11794	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11795	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11796	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11797	}
11798	else
11799	rc = VERR_EM_INTERPRETER;
11800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11801	return rc;
11802	}
11803
11804
11805	/**
11806	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11807	*/
11808	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11809	{
11810	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11811	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11812	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11813	AssertRCReturn(rc, rc);
11814
11815	PVM pVM = pVCpu->CTX_SUFF(pVM);
11816	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11817	if (RT_SUCCESS(rc))
11818	{
11819	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11820	Assert(pVmxTransient->cbInstr == 3);
11821	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11822	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11823	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11824	}
11825	else
11826	{
11827	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11828	rc = VERR_EM_INTERPRETER;
11829	}
11830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11831	return rc;
11832	}
11833
11834
11835	/**
11836	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11837	*/
11838	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11839	{
11840	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11841	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11842	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11843	AssertRCReturn(rc, rc);
11844
11845	PVM pVM = pVCpu->CTX_SUFF(pVM);
11846	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11847	if (RT_LIKELY(rc == VINF_SUCCESS))
11848	{
11849	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11850	Assert(pVmxTransient->cbInstr == 2);
11851	}
11852	else
11853	{
11854	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11855	rc = VERR_EM_INTERPRETER;
11856	}
11857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11858	return rc;
11859	}
11860
11861
11862	/**
11863	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11864	*/
11865	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11866	{
11867	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11869
11870	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11871	if (pVCpu->hm.s.fHypercallsEnabled)
11872	{
11873	#if 0
11874	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11875	#else
11876	/* Aggressive state sync. for now. */
11877	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11878	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11879	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11880	AssertRCReturn(rc, rc);
11881	#endif
11882
11883	/* Perform the hypercall. */
11884	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11885	if (rcStrict == VINF_SUCCESS)
11886	{
11887	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11888	AssertRCReturn(rc, rc);
11889	}
11890	else
11891	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11892	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11893	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11894
11895	/* If the hypercall changes anything other than guest's general-purpose registers,
11896	we would need to reload the guest changed bits here before VM-entry. */
11897	}
11898	else
11899	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11900
11901	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11902	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11903	{
11904	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11905	rcStrict = VINF_SUCCESS;
11906	}
11907
11908	return rcStrict;
11909	}
11910
11911
11912	/**
11913	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11914	*/
11915	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11916	{
11917	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11918	PVM pVM = pVCpu->CTX_SUFF(pVM);
11919	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11920
11921	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11922	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11923	AssertRCReturn(rc, rc);
11924
11925	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11926	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11927	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11928	else
11929	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11930	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11932	return rcStrict;
11933	}
11934
11935
11936	/**
11937	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11938	*/
11939	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11940	{
11941	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11942	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11943	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11944	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11945	AssertRCReturn(rc, rc);
11946
11947	PVM pVM = pVCpu->CTX_SUFF(pVM);
11948	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11949	if (RT_LIKELY(rc == VINF_SUCCESS))
11950	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11951	else
11952	{
11953	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11954	rc = VERR_EM_INTERPRETER;
11955	}
11956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11957	return rc;
11958	}
11959
11960
11961	/**
11962	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11963	*/
11964	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11965	{
11966	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11967	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11968	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11969	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11970	AssertRCReturn(rc, rc);
11971
11972	PVM pVM = pVCpu->CTX_SUFF(pVM);
11973	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11974	rc = VBOXSTRICTRC_VAL(rc2);
11975	if (RT_LIKELY( rc == VINF_SUCCESS
11976	\|\| rc == VINF_EM_HALT))
11977	{
11978	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11979	AssertRCReturn(rc3, rc3);
11980
11981	if ( rc == VINF_EM_HALT
11982	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11983	{
11984	rc = VINF_SUCCESS;
11985	}
11986	}
11987	else
11988	{
11989	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11990	rc = VERR_EM_INTERPRETER;
11991	}
11992	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11993	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11994	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11995	return rc;
11996	}
11997
11998
11999	/**
12000	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12001	*/
12002	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12003	{
12004	/*
12005	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12006	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12007	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
12008	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12009	*/
12010	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12011	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12012	HMVMX_RETURN_UNEXPECTED_EXIT();
12013	}
12014
12015
12016	/**
12017	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12018	*/
12019	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12020	{
12021	/*
12022	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12023	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12024	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12025	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12026	*/
12027	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12028	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12029	HMVMX_RETURN_UNEXPECTED_EXIT();
12030	}
12031
12032
12033	/**
12034	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12035	*/
12036	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12037	{
12038	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12039	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12040	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12041	HMVMX_RETURN_UNEXPECTED_EXIT();
12042	}
12043
12044
12045	/**
12046	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12047	*/
12048	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12049	{
12050	/*
12051	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12052	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12053	* See Intel spec. 25.3 "Other Causes of VM-exits".
12054	*/
12055	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12056	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12057	HMVMX_RETURN_UNEXPECTED_EXIT();
12058	}
12059
12060
12061	/**
12062	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12063	* VM-exit.
12064	*/
12065	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12066	{
12067	/*
12068	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12069	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12070	*
12071	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12072	* See Intel spec. "23.8 Restrictions on VMX operation".
12073	*/
12074	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12075	return VINF_SUCCESS;
12076	}
12077
12078
12079	/**
12080	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12081	* VM-exit.
12082	*/
12083	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12084	{
12085	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12086	return VINF_EM_RESET;
12087	}
12088
12089
12090	/**
12091	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12092	*/
12093	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12094	{
12095	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12096	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12097
12098	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12099	AssertRCReturn(rc, rc);
12100
12101	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12102	rc = VINF_SUCCESS;
12103	else
12104	rc = VINF_EM_HALT;
12105
12106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12107	if (rc != VINF_SUCCESS)
12108	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12109	return rc;
12110	}
12111
12112
12113	/**
12114	* VM-exit handler for instructions that result in a \#UD exception delivered to
12115	* the guest.
12116	*/
12117	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12118	{
12119	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12120	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12121	return VINF_SUCCESS;
12122	}
12123
12124
12125	/**
12126	* VM-exit handler for expiry of the VMX preemption timer.
12127	*/
12128	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12129	{
12130	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12131
12132	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12133	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12134
12135	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12136	PVM pVM = pVCpu->CTX_SUFF(pVM);
12137	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12138	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12139	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12140	}
12141
12142
12143	/**
12144	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12145	*/
12146	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12147	{
12148	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12149
12150	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12151	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12152	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12153	AssertRCReturn(rc, rc);
12154
12155	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12156	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12157
12158	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12159
12160	return rcStrict;
12161	}
12162
12163
12164	/**
12165	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12166	*/
12167	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12168	{
12169	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12170
12171	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
12172	/** @todo implement EMInterpretInvpcid() */
12173	return VERR_EM_INTERPRETER;
12174	}
12175
12176
12177	/**
12178	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12179	* Error VM-exit.
12180	*/
12181	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12182	{
12183	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12184	AssertRCReturn(rc, rc);
12185
12186	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12187	AssertRCReturn(rc, rc);
12188
12189	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12190	NOREF(uInvalidReason);
12191
12192	#ifdef VBOX_STRICT
12193	uint32_t uIntrState;
12194	RTHCUINTREG uHCReg;
12195	uint64_t u64Val;
12196	uint32_t u32Val;
12197
12198	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12199	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12200	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12201	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12202	AssertRCReturn(rc, rc);
12203
12204	Log4(("uInvalidReason %u\n", uInvalidReason));
12205	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12206	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12207	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12208	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12209
12210	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12211	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12212	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12213	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12214	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12215	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12216	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12217	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12218	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12219	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12220	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12221	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12222	#else
12223	NOREF(pVmxTransient);
12224	#endif
12225
12226	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12227	return VERR_VMX_INVALID_GUEST_STATE;
12228	}
12229
12230
12231	/**
12232	* VM-exit handler for VM-entry failure due to an MSR-load
12233	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12234	*/
12235	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12236	{
12237	NOREF(pVmxTransient);
12238	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12239	HMVMX_RETURN_UNEXPECTED_EXIT();
12240	}
12241
12242
12243	/**
12244	* VM-exit handler for VM-entry failure due to a machine-check event
12245	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12246	*/
12247	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12248	{
12249	NOREF(pVmxTransient);
12250	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12251	HMVMX_RETURN_UNEXPECTED_EXIT();
12252	}
12253
12254
12255	/**
12256	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12257	* theory.
12258	*/
12259	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12260	{
12261	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12262	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12263	return VERR_VMX_UNDEFINED_EXIT_CODE;
12264	}
12265
12266
12267	/**
12268	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12269	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12270	* Conditional VM-exit.
12271	*/
12272	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12273	{
12274	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12275
12276	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12278	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12279	return VERR_EM_INTERPRETER;
12280	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12281	HMVMX_RETURN_UNEXPECTED_EXIT();
12282	}
12283
12284
12285	/**
12286	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12287	*/
12288	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12289	{
12290	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12291
12292	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12293	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12294	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12295	return VERR_EM_INTERPRETER;
12296	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12297	HMVMX_RETURN_UNEXPECTED_EXIT();
12298	}
12299
12300
12301	/**
12302	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12303	*/
12304	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12305	{
12306	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12307
12308	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12309	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12310	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12311	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12312	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12313	{
12314	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12315	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12316	}
12317	AssertRCReturn(rc, rc);
12318	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12319
12320	#ifdef VBOX_STRICT
12321	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12322	{
12323	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12324	&& pMixedCtx->ecx != MSR_K6_EFER)
12325	{
12326	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12327	pMixedCtx->ecx));
12328	HMVMX_RETURN_UNEXPECTED_EXIT();
12329	}
12330	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12331	{
12332	VMXMSREXITREAD enmRead;
12333	VMXMSREXITWRITE enmWrite;
12334	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12335	AssertRCReturn(rc2, rc2);
12336	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12337	{
12338	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12339	HMVMX_RETURN_UNEXPECTED_EXIT();
12340	}
12341	}
12342	}
12343	#endif
12344
12345	PVM pVM = pVCpu->CTX_SUFF(pVM);
12346	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12347	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12348	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12350	if (RT_SUCCESS(rc))
12351	{
12352	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12353	Assert(pVmxTransient->cbInstr == 2);
12354	}
12355	return rc;
12356	}
12357
12358
12359	/**
12360	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12361	*/
12362	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12363	{
12364	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12365	PVM pVM = pVCpu->CTX_SUFF(pVM);
12366	int rc = VINF_SUCCESS;
12367
12368	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12369	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12370	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12371	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12372	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12373	{
12374	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12375	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12376	}
12377	AssertRCReturn(rc, rc);
12378	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12379
12380	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12381	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12383
12384	if (RT_SUCCESS(rc))
12385	{
12386	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12387
12388	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12389	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12390	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12391	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12392	{
12393	/*
12394	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12395	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12396	* EMInterpretWrmsr() changes it.
12397	*/
12398	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12399	}
12400	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12401	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12402	else if (pMixedCtx->ecx == MSR_K6_EFER)
12403	{
12404	/*
12405	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12406	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12407	* the other bits as well, SCE and NXE. See @bugref{7368}.
12408	*/
12409	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12410	}
12411
12412	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12413	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12414	{
12415	switch (pMixedCtx->ecx)
12416	{
12417	/*
12418	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12419	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12420	*/
12421	case MSR_IA32_SYSENTER_CS:
12422	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12423	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12424	break;
12425	case MSR_IA32_SYSENTER_EIP:
12426	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12427	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12428	break;
12429	case MSR_IA32_SYSENTER_ESP:
12430	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12431	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12432	break;
12433	case MSR_K8_FS_BASE: /* fall thru */
12434	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12435	case MSR_K6_EFER: /* already handled above */ break;
12436	default:
12437	{
12438	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12439	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12440	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12441	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12442	break;
12443	}
12444	}
12445	}
12446	#ifdef VBOX_STRICT
12447	else
12448	{
12449	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12450	switch (pMixedCtx->ecx)
12451	{
12452	case MSR_IA32_SYSENTER_CS:
12453	case MSR_IA32_SYSENTER_EIP:
12454	case MSR_IA32_SYSENTER_ESP:
12455	case MSR_K8_FS_BASE:
12456	case MSR_K8_GS_BASE:
12457	{
12458	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12459	HMVMX_RETURN_UNEXPECTED_EXIT();
12460	}
12461
12462	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12463	default:
12464	{
12465	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12466	{
12467	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12468	if (pMixedCtx->ecx != MSR_K6_EFER)
12469	{
12470	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12471	pMixedCtx->ecx));
12472	HMVMX_RETURN_UNEXPECTED_EXIT();
12473	}
12474	}
12475
12476	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12477	{
12478	VMXMSREXITREAD enmRead;
12479	VMXMSREXITWRITE enmWrite;
12480	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12481	AssertRCReturn(rc2, rc2);
12482	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12483	{
12484	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12485	HMVMX_RETURN_UNEXPECTED_EXIT();
12486	}
12487	}
12488	break;
12489	}
12490	}
12491	}
12492	#endif /* VBOX_STRICT */
12493	}
12494	return rc;
12495	}
12496
12497
12498	/**
12499	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12500	*/
12501	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12502	{
12503	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12504
12505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12506	return VINF_EM_RAW_INTERRUPT;
12507	}
12508
12509
12510	/**
12511	* VM-exit handler for when the TPR value is lowered below the specified
12512	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12513	*/
12514	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12515	{
12516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12517	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12518
12519	/*
12520	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12521	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12522	*/
12523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12524	return VINF_SUCCESS;
12525	}
12526
12527
12528	/**
12529	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12530	* VM-exit.
12531	*
12532	* @retval VINF_SUCCESS when guest execution can continue.
12533	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12534	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12535	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12536	* interpreter.
12537	*/
12538	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12539	{
12540	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12541	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12542	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12543	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12544	AssertRCReturn(rc, rc);
12545
12546	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12547	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12548	PVM pVM = pVCpu->CTX_SUFF(pVM);
12549	VBOXSTRICTRC rcStrict;
12550	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12551	switch (uAccessType)
12552	{
12553	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12554	{
12555	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12556	AssertRCReturn(rc, rc);
12557
12558	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12559	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12560	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12561	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12562	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12563	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12564	{
12565	case 0: /* CR0 */
12566	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12567	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12568	break;
12569	case 2: /* CR2 */
12570	/* Nothing to do here, CR2 it's not part of the VMCS. */
12571	break;
12572	case 3: /* CR3 */
12573	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12574	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12575	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12576	break;
12577	case 4: /* CR4 */
12578	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12579	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12580	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12581	break;
12582	case 8: /* CR8 */
12583	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12584	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12585	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12586	break;
12587	default:
12588	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12589	break;
12590	}
12591
12592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12593	break;
12594	}
12595
12596	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12597	{
12598	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12599	AssertRCReturn(rc, rc);
12600
12601	Assert( !pVM->hm.s.fNestedPaging
12602	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12603	\|\| pVCpu->hm.s.fUsingDebugLoop
12604	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12605
12606	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12607	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12608	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12609
12610	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12611	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12612	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12613	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12614	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12615	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12616	VBOXSTRICTRC_VAL(rcStrict)));
12617	break;
12618	}
12619
12620	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12621	{
12622	AssertRCReturn(rc, rc);
12623	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12624	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12625	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12626	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12627	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12628	break;
12629	}
12630
12631	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12632	{
12633	AssertRCReturn(rc, rc);
12634	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12635	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12636	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12637	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12638	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12640	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12641	break;
12642	}
12643
12644	default:
12645	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12646	VERR_VMX_UNEXPECTED_EXCEPTION);
12647	}
12648
12649	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12650	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12651	NOREF(pVM);
12652	return rcStrict;
12653	}
12654
12655
12656	/**
12657	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12658	* VM-exit.
12659	*/
12660	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12661	{
12662	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12663	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12664
12665	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12666	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12667	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12668	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12669	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12670	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12671	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12672	AssertRCReturn(rc, rc);
12673
12674	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12675	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12676	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12677	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12678	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12679	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12680	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12681	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12682	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12683
12684	/* I/O operation lookup arrays. */
12685	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12686	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12687
12688	VBOXSTRICTRC rcStrict;
12689	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12690	uint32_t const cbInstr = pVmxTransient->cbInstr;
12691	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12692	PVM pVM = pVCpu->CTX_SUFF(pVM);
12693	if (fIOString)
12694	{
12695	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12696	See @bugref{5752#c158}. Should work now. */
12697	/*
12698	* INS/OUTS - I/O String instruction.
12699	*
12700	* Use instruction-information if available, otherwise fall back on
12701	* interpreting the instruction.
12702	*/
12703	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12704	fIOWrite ? 'w' : 'r'));
12705	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12706	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12707	{
12708	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12709	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12710	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12711	AssertRCReturn(rc2, rc2);
12712	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12713	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12714	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12715	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12716	if (fIOWrite)
12717	{
12718	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12719	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12720	}
12721	else
12722	{
12723	/*
12724	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12725	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12726	* See Intel Instruction spec. for "INS".
12727	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12728	*/
12729	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12730	}
12731	}
12732	else
12733	{
12734	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12735	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12736	AssertRCReturn(rc2, rc2);
12737	rcStrict = IEMExecOne(pVCpu);
12738	}
12739	/** @todo IEM needs to be setting these flags somehow. */
12740	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12741	fUpdateRipAlready = true;
12742	#else
12743	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12744	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12745	if (RT_SUCCESS(rcStrict))
12746	{
12747	if (fIOWrite)
12748	{
12749	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12750	(DISCPUMODE)pDis->uAddrMode, cbValue);
12751	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12752	}
12753	else
12754	{
12755	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12756	(DISCPUMODE)pDis->uAddrMode, cbValue);
12757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12758	}
12759	}
12760	else
12761	{
12762	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12763	pMixedCtx->rip));
12764	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12765	}
12766	#endif
12767	}
12768	else
12769	{
12770	/*
12771	* IN/OUT - I/O instruction.
12772	*/
12773	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12774	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12775	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12776	if (fIOWrite)
12777	{
12778	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12779	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12780	}
12781	else
12782	{
12783	uint32_t u32Result = 0;
12784	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12785	if (IOM_SUCCESS(rcStrict))
12786	{
12787	/* Save result of I/O IN instr. in AL/AX/EAX. */
12788	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12789	}
12790	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12791	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12793	}
12794	}
12795
12796	if (IOM_SUCCESS(rcStrict))
12797	{
12798	if (!fUpdateRipAlready)
12799	{
12800	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12801	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12802	}
12803
12804	/*
12805	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12806	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12807	*/
12808	if (fIOString)
12809	{
12810	/** @todo Single-step for INS/OUTS with REP prefix? */
12811	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12812	}
12813	else if ( !fDbgStepping
12814	&& fGstStepping)
12815	{
12816	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12817	}
12818
12819	/*
12820	* If any I/O breakpoints are armed, we need to check if one triggered
12821	* and take appropriate action.
12822	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12823	*/
12824	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12825	AssertRCReturn(rc2, rc2);
12826
12827	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12828	* execution engines about whether hyper BPs and such are pending. */
12829	uint32_t const uDr7 = pMixedCtx->dr[7];
12830	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12831	&& X86_DR7_ANY_RW_IO(uDr7)
12832	&& (pMixedCtx->cr4 & X86_CR4_DE))
12833	\|\| DBGFBpIsHwIoArmed(pVM)))
12834	{
12835	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12836
12837	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12838	VMMRZCallRing3Disable(pVCpu);
12839	HM_DISABLE_PREEMPT();
12840
12841	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12842
12843	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12844	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12845	{
12846	/* Raise #DB. */
12847	if (fIsGuestDbgActive)
12848	ASMSetDR6(pMixedCtx->dr[6]);
12849	if (pMixedCtx->dr[7] != uDr7)
12850	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12851
12852	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12853	}
12854	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12855	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12856	else if ( rcStrict2 != VINF_SUCCESS
12857	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12858	rcStrict = rcStrict2;
12859	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12860
12861	HM_RESTORE_PREEMPT();
12862	VMMRZCallRing3Enable(pVCpu);
12863	}
12864	}
12865
12866	#ifdef VBOX_STRICT
12867	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12868	Assert(!fIOWrite);
12869	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12870	Assert(fIOWrite);
12871	else
12872	{
12873	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12874	* statuses, that the VMM device and some others may return. See
12875	* IOM_SUCCESS() for guidance. */
12876	AssertMsg( RT_FAILURE(rcStrict)
12877	\|\| rcStrict == VINF_SUCCESS
12878	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12879	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12880	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12881	\|\| rcStrict == VINF_EM_RAW_TO_R3
12882	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12883	#endif
12884	}
12885	#endif
12886
12887	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12888	return rcStrict;
12889	}
12890
12891
12892	/**
12893	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12894	* VM-exit.
12895	*/
12896	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12897	{
12898	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12899
12900	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12901	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12902	AssertRCReturn(rc, rc);
12903	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12904	{
12905	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12906	AssertRCReturn(rc, rc);
12907	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12908	{
12909	uint32_t uErrCode;
12910	RTGCUINTPTR GCPtrFaultAddress;
12911	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12912	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12913	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12914	if (fErrorCodeValid)
12915	{
12916	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12917	AssertRCReturn(rc, rc);
12918	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12919	}
12920	else
12921	uErrCode = 0;
12922
12923	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12924	&& uVector == X86_XCPT_PF)
12925	GCPtrFaultAddress = pMixedCtx->cr2;
12926	else
12927	GCPtrFaultAddress = 0;
12928
12929	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12930	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12931
12932	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12934	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12935	}
12936	}
12937
12938	/* Fall back to the interpreter to emulate the task-switch. */
12939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12940	return VERR_EM_INTERPRETER;
12941	}
12942
12943
12944	/**
12945	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12946	*/
12947	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12948	{
12949	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12950	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12951	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12953	AssertRCReturn(rc, rc);
12954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12955	return VINF_EM_DBG_STEPPED;
12956	}
12957
12958
12959	/**
12960	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12961	*/
12962	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12963	{
12964	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12965
12966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12967
12968	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12969	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12970	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12971	{
12972	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12973	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12974	{
12975	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12976	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12977	}
12978	}
12979	else
12980	{
12981	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12982	rcStrict1 = VINF_SUCCESS;
12983	return rcStrict1;
12984	}
12985
12986	#if 0
12987	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12988	* just sync the whole thing. */
12989	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12990	#else
12991	/* Aggressive state sync. for now. */
12992	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12993	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12994	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12995	#endif
12996	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12997	AssertRCReturn(rc, rc);
12998
12999	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13000	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13001	VBOXSTRICTRC rcStrict2;
13002	switch (uAccessType)
13003	{
13004	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13005	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13006	{
13007	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13008	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13009	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13010
13011	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13012	GCPhys &= PAGE_BASE_GC_MASK;
13013	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13014	PVM pVM = pVCpu->CTX_SUFF(pVM);
13015	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13016	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13017
13018	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13019	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13020	CPUMCTX2CORE(pMixedCtx), GCPhys);
13021	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13022	if ( rcStrict2 == VINF_SUCCESS
13023	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13024	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13025	{
13026	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13027	\| HM_CHANGED_GUEST_RSP
13028	\| HM_CHANGED_GUEST_RFLAGS
13029	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13030	rcStrict2 = VINF_SUCCESS;
13031	}
13032	break;
13033	}
13034
13035	default:
13036	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13037	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13038	break;
13039	}
13040
13041	if (rcStrict2 != VINF_SUCCESS)
13042	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13043	return rcStrict2;
13044	}
13045
13046
13047	/**
13048	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13049	* VM-exit.
13050	*/
13051	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13052	{
13053	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13054
13055	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13056	if (pVmxTransient->fWasGuestDebugStateActive)
13057	{
13058	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13059	HMVMX_RETURN_UNEXPECTED_EXIT();
13060	}
13061
13062	if ( !pVCpu->hm.s.fSingleInstruction
13063	&& !pVmxTransient->fWasHyperDebugStateActive)
13064	{
13065	Assert(!DBGFIsStepping(pVCpu));
13066	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13067
13068	/* Don't intercept MOV DRx any more. */
13069	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13070	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13071	AssertRCReturn(rc, rc);
13072
13073	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13074	VMMRZCallRing3Disable(pVCpu);
13075	HM_DISABLE_PREEMPT();
13076
13077	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13078	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13079	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13080
13081	HM_RESTORE_PREEMPT();
13082	VMMRZCallRing3Enable(pVCpu);
13083
13084	#ifdef VBOX_WITH_STATISTICS
13085	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13086	AssertRCReturn(rc, rc);
13087	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13088	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13089	else
13090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13091	#endif
13092	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13093	return VINF_SUCCESS;
13094	}
13095
13096	/*
13097	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13098	* Update the segment registers and DR7 from the CPU.
13099	*/
13100	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13101	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13102	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13103	AssertRCReturn(rc, rc);
13104	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13105
13106	PVM pVM = pVCpu->CTX_SUFF(pVM);
13107	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13108	{
13109	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13110	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13111	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13112	if (RT_SUCCESS(rc))
13113	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13114	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13115	}
13116	else
13117	{
13118	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13119	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13120	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13121	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13122	}
13123
13124	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13125	if (RT_SUCCESS(rc))
13126	{
13127	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13128	AssertRCReturn(rc2, rc2);
13129	return VINF_SUCCESS;
13130	}
13131	return rc;
13132	}
13133
13134
13135	/**
13136	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13137	* Conditional VM-exit.
13138	*/
13139	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13140	{
13141	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13142	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13143
13144	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13145	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13146	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13147	{
13148	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13149	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13150	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13151	{
13152	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13153	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13154	}
13155	}
13156	else
13157	{
13158	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13159	rcStrict1 = VINF_SUCCESS;
13160	return rcStrict1;
13161	}
13162
13163	RTGCPHYS GCPhys = 0;
13164	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13165
13166	#if 0
13167	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13168	#else
13169	/* Aggressive state sync. for now. */
13170	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13171	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13172	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13173	#endif
13174	AssertRCReturn(rc, rc);
13175
13176	/*
13177	* If we succeed, resume guest execution.
13178	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13179	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13180	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13181	* weird case. See @bugref{6043}.
13182	*/
13183	PVM pVM = pVCpu->CTX_SUFF(pVM);
13184	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13185	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13186	if ( rcStrict2 == VINF_SUCCESS
13187	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13188	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13189	{
13190	/* Successfully handled MMIO operation. */
13191	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13192	\| HM_CHANGED_GUEST_RSP
13193	\| HM_CHANGED_GUEST_RFLAGS
13194	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13195	return VINF_SUCCESS;
13196	}
13197	return rcStrict2;
13198	}
13199
13200
13201	/**
13202	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13203	* VM-exit.
13204	*/
13205	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13206	{
13207	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13208	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13209
13210	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13211	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13212	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13213	{
13214	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13215	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13216	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13217	}
13218	else
13219	{
13220	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13221	rcStrict1 = VINF_SUCCESS;
13222	return rcStrict1;
13223	}
13224
13225	RTGCPHYS GCPhys = 0;
13226	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13227	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13228	#if 0
13229	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13230	#else
13231	/* Aggressive state sync. for now. */
13232	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13233	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13234	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13235	#endif
13236	AssertRCReturn(rc, rc);
13237
13238	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13239	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13240
13241	RTGCUINT uErrorCode = 0;
13242	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13243	uErrorCode \|= X86_TRAP_PF_ID;
13244	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13245	uErrorCode \|= X86_TRAP_PF_RW;
13246	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13247	uErrorCode \|= X86_TRAP_PF_P;
13248
13249	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13250
13251	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13252	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13253
13254	/* Handle the pagefault trap for the nested shadow table. */
13255	PVM pVM = pVCpu->CTX_SUFF(pVM);
13256	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13257	TRPMResetTrap(pVCpu);
13258
13259	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13260	if ( rcStrict2 == VINF_SUCCESS
13261	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13262	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13263	{
13264	/* Successfully synced our nested page tables. */
13265	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13266	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13267	\| HM_CHANGED_GUEST_RSP
13268	\| HM_CHANGED_GUEST_RFLAGS);
13269	return VINF_SUCCESS;
13270	}
13271
13272	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13273	return rcStrict2;
13274	}
13275
13276	/** @} */
13277
13278	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13279	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13280	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13281
13282	/** @name VM-exit exception handlers.
13283	* @{
13284	*/
13285
13286	/**
13287	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13288	*/
13289	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13290	{
13291	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13292	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13293
13294	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13295	AssertRCReturn(rc, rc);
13296
13297	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13298	{
13299	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13300	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13301
13302	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13303	* provides VM-exit instruction length. If this causes problem later,
13304	* disassemble the instruction like it's done on AMD-V. */
13305	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13306	AssertRCReturn(rc2, rc2);
13307	return rc;
13308	}
13309
13310	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13311	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13312	return rc;
13313	}
13314
13315
13316	/**
13317	* VM-exit exception handler for \#BP (Breakpoint exception).
13318	*/
13319	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13320	{
13321	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13322	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13323
13324	/** @todo Try optimize this by not saving the entire guest state unless
13325	* really needed. */
13326	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13327	AssertRCReturn(rc, rc);
13328
13329	PVM pVM = pVCpu->CTX_SUFF(pVM);
13330	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13331	if (rc == VINF_EM_RAW_GUEST_TRAP)
13332	{
13333	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13334	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13335	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13336	AssertRCReturn(rc, rc);
13337
13338	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13339	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13340	}
13341
13342	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13343	return rc;
13344	}
13345
13346
13347	/**
13348	* VM-exit exception handler for \#AC (alignment check exception).
13349	*/
13350	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13351	{
13352	RT_NOREF_PV(pMixedCtx);
13353	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13354
13355	/*
13356	* Re-inject it. We'll detect any nesting before getting here.
13357	*/
13358	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13359	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13360	AssertRCReturn(rc, rc);
13361	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13362
13363	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13364	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13365	return VINF_SUCCESS;
13366	}
13367
13368
13369	/**
13370	* VM-exit exception handler for \#DB (Debug exception).
13371	*/
13372	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13373	{
13374	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13376	Log6(("XcptDB\n"));
13377
13378	/*
13379	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13380	* for processing.
13381	*/
13382	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13383	AssertRCReturn(rc, rc);
13384
13385	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13386	uint64_t uDR6 = X86_DR6_INIT_VAL;
13387	uDR6 \|= ( pVmxTransient->uExitQualification
13388	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13389
13390	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13391	if (rc == VINF_EM_RAW_GUEST_TRAP)
13392	{
13393	/*
13394	* The exception was for the guest. Update DR6, DR7.GD and
13395	* IA32_DEBUGCTL.LBR before forwarding it.
13396	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13397	*/
13398	VMMRZCallRing3Disable(pVCpu);
13399	HM_DISABLE_PREEMPT();
13400
13401	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13402	pMixedCtx->dr[6] \|= uDR6;
13403	if (CPUMIsGuestDebugStateActive(pVCpu))
13404	ASMSetDR6(pMixedCtx->dr[6]);
13405
13406	HM_RESTORE_PREEMPT();
13407	VMMRZCallRing3Enable(pVCpu);
13408
13409	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13410	AssertRCReturn(rc, rc);
13411
13412	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13413	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13414
13415	/* Paranoia. */
13416	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13417	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13418
13419	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13420	AssertRCReturn(rc, rc);
13421
13422	/*
13423	* Raise #DB in the guest.
13424	*
13425	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13426	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13427	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13428	*
13429	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13430	*/
13431	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13432	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13433	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13434	AssertRCReturn(rc, rc);
13435	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13436	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13437	return VINF_SUCCESS;
13438	}
13439
13440	/*
13441	* Not a guest trap, must be a hypervisor related debug event then.
13442	* Update DR6 in case someone is interested in it.
13443	*/
13444	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13445	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13446	CPUMSetHyperDR6(pVCpu, uDR6);
13447
13448	return rc;
13449	}
13450
13451
13452	/**
13453	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13454	* point exception).
13455	*/
13456	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13457	{
13458	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13459
13460	/* We require CR0 and EFER. EFER is always up-to-date. */
13461	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13462	AssertRCReturn(rc, rc);
13463
13464	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13465	VMMRZCallRing3Disable(pVCpu);
13466	HM_DISABLE_PREEMPT();
13467
13468	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13469	if (pVmxTransient->fWasGuestFPUStateActive)
13470	{
13471	rc = VINF_EM_RAW_GUEST_TRAP;
13472	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13473	}
13474	else
13475	{
13476	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13477	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13478	#endif
13479	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13480	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13481	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13482	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13483	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13484	}
13485
13486	HM_RESTORE_PREEMPT();
13487	VMMRZCallRing3Enable(pVCpu);
13488
13489	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13490	{
13491	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13492	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13494	pVCpu->hm.s.fPreloadGuestFpu = true;
13495	}
13496	else
13497	{
13498	/* Forward #NM to the guest. */
13499	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13500	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13501	AssertRCReturn(rc, rc);
13502	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13503	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13505	}
13506
13507	return VINF_SUCCESS;
13508	}
13509
13510
13511	/**
13512	* VM-exit exception handler for \#GP (General-protection exception).
13513	*
13514	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13515	*/
13516	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13517	{
13518	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13519	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13520
13521	int rc;
13522	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13523	{ /* likely */ }
13524	else
13525	{
13526	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13527	Assert(pVCpu->hm.s.fUsingDebugLoop);
13528	#endif
13529	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13530	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13531	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13532	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13533	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13534	AssertRCReturn(rc, rc);
13535	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13536	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13537	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13538	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13539	return rc;
13540	}
13541
13542	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13543	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13544
13545	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13546	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13547	AssertRCReturn(rc, rc);
13548
13549	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13550	uint32_t cbOp = 0;
13551	PVM pVM = pVCpu->CTX_SUFF(pVM);
13552	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13553	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13554	if (RT_SUCCESS(rc))
13555	{
13556	rc = VINF_SUCCESS;
13557	Assert(cbOp == pDis->cbInstr);
13558	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13559	switch (pDis->pCurInstr->uOpcode)
13560	{
13561	case OP_CLI:
13562	{
13563	pMixedCtx->eflags.Bits.u1IF = 0;
13564	pMixedCtx->eflags.Bits.u1RF = 0;
13565	pMixedCtx->rip += pDis->cbInstr;
13566	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13567	if ( !fDbgStepping
13568	&& pMixedCtx->eflags.Bits.u1TF)
13569	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13571	break;
13572	}
13573
13574	case OP_STI:
13575	{
13576	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13577	pMixedCtx->eflags.Bits.u1IF = 1;
13578	pMixedCtx->eflags.Bits.u1RF = 0;
13579	pMixedCtx->rip += pDis->cbInstr;
13580	if (!fOldIF)
13581	{
13582	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13583	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13584	}
13585	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13586	if ( !fDbgStepping
13587	&& pMixedCtx->eflags.Bits.u1TF)
13588	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13589	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13590	break;
13591	}
13592
13593	case OP_HLT:
13594	{
13595	rc = VINF_EM_HALT;
13596	pMixedCtx->rip += pDis->cbInstr;
13597	pMixedCtx->eflags.Bits.u1RF = 0;
13598	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13600	break;
13601	}
13602
13603	case OP_POPF:
13604	{
13605	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13606	uint32_t cbParm;
13607	uint32_t uMask;
13608	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13609	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13610	{
13611	cbParm = 4;
13612	uMask = 0xffffffff;
13613	}
13614	else
13615	{
13616	cbParm = 2;
13617	uMask = 0xffff;
13618	}
13619
13620	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13621	RTGCPTR GCPtrStack = 0;
13622	X86EFLAGS Eflags;
13623	Eflags.u32 = 0;
13624	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13625	&GCPtrStack);
13626	if (RT_SUCCESS(rc))
13627	{
13628	Assert(sizeof(Eflags.u32) >= cbParm);
13629	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13630	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13631	}
13632	if (RT_FAILURE(rc))
13633	{
13634	rc = VERR_EM_INTERPRETER;
13635	break;
13636	}
13637	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13638	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13639	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13640	pMixedCtx->esp += cbParm;
13641	pMixedCtx->esp &= uMask;
13642	pMixedCtx->rip += pDis->cbInstr;
13643	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13644	\| HM_CHANGED_GUEST_RSP
13645	\| HM_CHANGED_GUEST_RFLAGS);
13646	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13647	POPF restores EFLAGS.TF. */
13648	if ( !fDbgStepping
13649	&& fGstStepping)
13650	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13652	break;
13653	}
13654
13655	case OP_PUSHF:
13656	{
13657	uint32_t cbParm;
13658	uint32_t uMask;
13659	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13660	{
13661	cbParm = 4;
13662	uMask = 0xffffffff;
13663	}
13664	else
13665	{
13666	cbParm = 2;
13667	uMask = 0xffff;
13668	}
13669
13670	/* Get the stack pointer & push the contents of eflags onto the stack. */
13671	RTGCPTR GCPtrStack = 0;
13672	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13673	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13674	if (RT_FAILURE(rc))
13675	{
13676	rc = VERR_EM_INTERPRETER;
13677	break;
13678	}
13679	X86EFLAGS Eflags = pMixedCtx->eflags;
13680	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13681	Eflags.Bits.u1RF = 0;
13682	Eflags.Bits.u1VM = 0;
13683
13684	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13685	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13686	{
13687	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13688	rc = VERR_EM_INTERPRETER;
13689	break;
13690	}
13691	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13692	pMixedCtx->esp -= cbParm;
13693	pMixedCtx->esp &= uMask;
13694	pMixedCtx->rip += pDis->cbInstr;
13695	pMixedCtx->eflags.Bits.u1RF = 0;
13696	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13697	\| HM_CHANGED_GUEST_RSP
13698	\| HM_CHANGED_GUEST_RFLAGS);
13699	if ( !fDbgStepping
13700	&& pMixedCtx->eflags.Bits.u1TF)
13701	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13702	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13703	break;
13704	}
13705
13706	case OP_IRET:
13707	{
13708	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13709	* instruction reference. */
13710	RTGCPTR GCPtrStack = 0;
13711	uint32_t uMask = 0xffff;
13712	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13713	uint16_t aIretFrame[3];
13714	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13715	{
13716	rc = VERR_EM_INTERPRETER;
13717	break;
13718	}
13719	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13720	&GCPtrStack);
13721	if (RT_SUCCESS(rc))
13722	{
13723	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13724	PGMACCESSORIGIN_HM));
13725	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13726	}
13727	if (RT_FAILURE(rc))
13728	{
13729	rc = VERR_EM_INTERPRETER;
13730	break;
13731	}
13732	pMixedCtx->eip = 0;
13733	pMixedCtx->ip = aIretFrame[0];
13734	pMixedCtx->cs.Sel = aIretFrame[1];
13735	pMixedCtx->cs.ValidSel = aIretFrame[1];
13736	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13737	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13738	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13739	pMixedCtx->sp += sizeof(aIretFrame);
13740	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13741	\| HM_CHANGED_GUEST_SEGMENT_REGS
13742	\| HM_CHANGED_GUEST_RSP
13743	\| HM_CHANGED_GUEST_RFLAGS);
13744	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13745	if ( !fDbgStepping
13746	&& fGstStepping)
13747	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13748	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13749	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13750	break;
13751	}
13752
13753	case OP_INT:
13754	{
13755	uint16_t uVector = pDis->Param1.uValue & 0xff;
13756	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13757	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13759	break;
13760	}
13761
13762	case OP_INTO:
13763	{
13764	if (pMixedCtx->eflags.Bits.u1OF)
13765	{
13766	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13767	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13768	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13769	}
13770	else
13771	{
13772	pMixedCtx->eflags.Bits.u1RF = 0;
13773	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13774	}
13775	break;
13776	}
13777
13778	default:
13779	{
13780	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13781	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13782	EMCODETYPE_SUPERVISOR);
13783	rc = VBOXSTRICTRC_VAL(rc2);
13784	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13785	/** @todo We have to set pending-debug exceptions here when the guest is
13786	* single-stepping depending on the instruction that was interpreted. */
13787	Log4(("#GP rc=%Rrc\n", rc));
13788	break;
13789	}
13790	}
13791	}
13792	else
13793	rc = VERR_EM_INTERPRETER;
13794
13795	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13796	("#GP Unexpected rc=%Rrc\n", rc));
13797	return rc;
13798	}
13799
13800
13801	/**
13802	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13803	* the exception reported in the VMX transient structure back into the VM.
13804	*
13805	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13806	* up-to-date.
13807	*/
13808	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13809	{
13810	RT_NOREF_PV(pMixedCtx);
13811	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13812	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13813	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13814	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13815	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13816	#endif
13817
13818	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13819	hmR0VmxCheckExitDueToEventDelivery(). */
13820	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13821	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13822	AssertRCReturn(rc, rc);
13823	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13824
13825	#ifdef DEBUG_ramshankar
13826	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13827	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13828	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13829	#endif
13830
13831	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13832	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13833	return VINF_SUCCESS;
13834	}
13835
13836
13837	/**
13838	* VM-exit exception handler for \#PF (Page-fault exception).
13839	*/
13840	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13841	{
13842	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13843	PVM pVM = pVCpu->CTX_SUFF(pVM);
13844	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13845	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13846	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13847	AssertRCReturn(rc, rc);
13848
13849	if (!pVM->hm.s.fNestedPaging)
13850	{ /* likely */ }
13851	else
13852	{
13853	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13854	Assert(pVCpu->hm.s.fUsingDebugLoop);
13855	#endif
13856	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13857	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13858	{
13859	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13860	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13861	}
13862	else
13863	{
13864	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13865	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13866	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13867	}
13868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13869	return rc;
13870	}
13871
13872	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13873	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13874	if (pVmxTransient->fVectoringPF)
13875	{
13876	Assert(pVCpu->hm.s.Event.fPending);
13877	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13878	}
13879
13880	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13881	AssertRCReturn(rc, rc);
13882
13883	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13884	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13885
13886	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13887	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13888	(RTGCPTR)pVmxTransient->uExitQualification);
13889
13890	Log4(("#PF: rc=%Rrc\n", rc));
13891	if (rc == VINF_SUCCESS)
13892	{
13893	#if 0
13894	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13895	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13896	* memory? We don't update the whole state here... */
13897	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13898	\| HM_CHANGED_GUEST_RSP
13899	\| HM_CHANGED_GUEST_RFLAGS
13900	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13901	#else
13902	/*
13903	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13904	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13905	*/
13906	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13907	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13908	#endif
13909	TRPMResetTrap(pVCpu);
13910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13911	return rc;
13912	}
13913
13914	if (rc == VINF_EM_RAW_GUEST_TRAP)
13915	{
13916	if (!pVmxTransient->fVectoringDoublePF)
13917	{
13918	/* It's a guest page fault and needs to be reflected to the guest. */
13919	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13920	TRPMResetTrap(pVCpu);
13921	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13922	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13923	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13924	}
13925	else
13926	{
13927	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13928	TRPMResetTrap(pVCpu);
13929	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13930	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13931	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13932	}
13933
13934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13935	return VINF_SUCCESS;
13936	}
13937
13938	TRPMResetTrap(pVCpu);
13939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13940	return rc;
13941	}
13942
13943	/** @} */
13944

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 67035

Download in other formats: