HMVMXR0.cpp@ 61605

Last change on this file since 61605 was 61544, checked in by vboxsync, 9 years ago
VMM/GIM: Fix up hypercall rc handling and also a bug while disassembling hypercalls. IEM/REM being able to handle VMCALL/VMMCALL to hook into GIM is still a todo but this change temporarily works around by adding new GIM and GIM provider interfaces to kinda do the same work. The Xcpt handlers were not re-used as they check for whether or not the provider requested UD trapping in the first place.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 576.7 KB

Line
1	/* $Id: HMVMXR0.cpp 61544 2016-06-07 14:42:20Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef VBOX_WITH_NEW_APIC
38	# include <VBox/vmm/apic.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include "HMVMXR0.h"
43	#include "dtrace/VBoxVMM.h"
44
45	#ifdef DEBUG_ramshankar
46	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51	# define HMVMX_ALWAYS_TRAP_PF
52	# define HMVMX_ALWAYS_SWAP_FPU_STATE
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name Updated-guest-state flags.
71	* @{ */
72	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93	\| HMVMX_UPDATED_GUEST_RSP \
94	\| HMVMX_UPDATED_GUEST_RFLAGS \
95	\| HMVMX_UPDATED_GUEST_CR0 \
96	\| HMVMX_UPDATED_GUEST_CR3 \
97	\| HMVMX_UPDATED_GUEST_CR4 \
98	\| HMVMX_UPDATED_GUEST_GDTR \
99	\| HMVMX_UPDATED_GUEST_IDTR \
100	\| HMVMX_UPDATED_GUEST_LDTR \
101	\| HMVMX_UPDATED_GUEST_TR \
102	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103	\| HMVMX_UPDATED_GUEST_DEBUG \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
109	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110	\| HMVMX_UPDATED_GUEST_INTR_STATE \
111	\| HMVMX_UPDATED_GUEST_APIC_STATE)
112	/** @} */
113
114	/** @name
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125	/** @} */
126
127	/** @name
128	* States of the VMCS.
129	*
130	* This does not reflect all possible VMCS states but currently only those
131	* needed for maintaining the VMCS consistently even when thread-context hooks
132	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
133	*/
134	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137	/** @} */
138
139	/**
140	* Exception bitmap mask for real-mode guests (real-on-v86).
141	*
142	* We need to intercept all exceptions manually except:
143	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145	* due to bugs in Intel CPUs.
146	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
147	* support.
148	*/
149	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
150	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
151	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
152	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
153	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
155	\| RT_BIT(X86_XCPT_XF))
156
157	/**
158	* Exception bitmap mask for all contributory exceptions.
159	*
160	* Page fault is deliberately excluded here as it's conditional as to whether
161	* it's contributory or benign. Page faults are handled separately.
162	*/
163	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
164	\| RT_BIT(X86_XCPT_DE))
165
166	/** Maximum VM-instruction error number. */
167	#define HMVMX_INSTR_ERROR_MAX 28
168
169	/** Profiling macro. */
170	#ifdef HM_PROFILE_EXIT_DISPATCH
171	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173	#else
174	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176	#endif
177
178	/** Assert that preemption is disabled or covered by thread-context hooks. */
179	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182	/** Assert that we haven't migrated CPUs when thread-context hooks are not
183	* used. */
184	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186	("Illegal migration! Entered on CPU %u Current %u\n", \
187	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189	/** Helper macro for VM-exit handlers called unexpectedly. */
190	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191	do { \
192	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193	return VERR_VMX_UNEXPECTED_EXIT; \
194	} while (0)
195
196
197	/*********************************************************************************************************************************
198	* Structures and Typedefs *
199	*********************************************************************************************************************************/
200	/**
201	* VMX transient state.
202	*
203	* A state structure for holding miscellaneous information across
204	* VMX non-root operation and restored after the transition.
205	*/
206	typedef struct VMXTRANSIENT
207	{
208	/** The host's rflags/eflags. */
209	RTCCUINTREG fEFlags;
210	#if HC_ARCH_BITS == 32
211	uint32_t u32Alignment0;
212	#endif
213	/** The guest's TPR value used for TPR shadowing. */
214	uint8_t u8GuestTpr;
215	/** Alignment. */
216	uint8_t abAlignment0[7];
217
218	/** The basic VM-exit reason. */
219	uint16_t uExitReason;
220	/** Alignment. */
221	uint16_t u16Alignment0;
222	/** The VM-exit interruption error code. */
223	uint32_t uExitIntErrorCode;
224	/** The VM-exit exit code qualification. */
225	uint64_t uExitQualification;
226
227	/** The VM-exit interruption-information field. */
228	uint32_t uExitIntInfo;
229	/** The VM-exit instruction-length field. */
230	uint32_t cbInstr;
231	/** The VM-exit instruction-information field. */
232	union
233	{
234	/** Plain unsigned int representation. */
235	uint32_t u;
236	/** INS and OUTS information. */
237	struct
238	{
239	uint32_t u7Reserved0 : 7;
240	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241	uint32_t u3AddrSize : 3;
242	uint32_t u5Reserved1 : 5;
243	/** The segment register (X86_SREG_XXX). */
244	uint32_t iSegReg : 3;
245	uint32_t uReserved2 : 14;
246	} StrIo;
247	} ExitInstrInfo;
248	/** Whether the VM-entry failed or not. */
249	bool fVMEntryFailed;
250	/** Alignment. */
251	uint8_t abAlignment1[3];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest FPU was active at the time of VM-exit. */
269	bool fWasGuestFPUStateActive;
270	/** Whether the guest debug state was active at the time of VM-exit. */
271	bool fWasGuestDebugStateActive;
272	/** Whether the hyper debug state was active at the time of VM-exit. */
273	bool fWasHyperDebugStateActive;
274	/** Whether TSC-offsetting should be setup before VM-entry. */
275	bool fUpdateTscOffsettingAndPreemptTimer;
276	/** Whether the VM-exit was caused by a page-fault during delivery of a
277	* contributory exception or a page-fault. */
278	bool fVectoringDoublePF;
279	/** Whether the VM-exit was caused by a page-fault during delivery of an
280	* external interrupt or NMI. */
281	bool fVectoringPF;
282	} VMXTRANSIENT;
283	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288	/** Pointer to VMX transient state. */
289	typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292	/**
293	* MSR-bitmap read permissions.
294	*/
295	typedef enum VMXMSREXITREAD
296	{
297	/** Reading this MSR causes a VM-exit. */
298	VMXMSREXIT_INTERCEPT_READ = 0xb,
299	/** Reading this MSR does not cause a VM-exit. */
300	VMXMSREXIT_PASSTHRU_READ
301	} VMXMSREXITREAD;
302	/** Pointer to MSR-bitmap read permissions. */
303	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305	/**
306	* MSR-bitmap write permissions.
307	*/
308	typedef enum VMXMSREXITWRITE
309	{
310	/** Writing to this MSR causes a VM-exit. */
311	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312	/** Writing to this MSR does not cause a VM-exit. */
313	VMXMSREXIT_PASSTHRU_WRITE
314	} VMXMSREXITWRITE;
315	/** Pointer to MSR-bitmap write permissions. */
316	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319	/**
320	* VMX VM-exit handler.
321	*
322	* @returns Strict VBox status code (i.e. informational status codes too).
323	* @param pVCpu The cross context virtual CPU structure.
324	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
325	* out-of-sync. Make sure to update the required
326	* fields before using them.
327	* @param pVmxTransient Pointer to the VMX-transient structure.
328	*/
329	#ifndef HMVMX_USE_FUNCTION_TABLE
330	typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331	#else
332	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333	/** Pointer to VM-exit handler. */
334	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335	#endif
336
337	/**
338	* VMX VM-exit handler, non-strict status code.
339	*
340	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341	*
342	* @returns VBox status code, no informational status code returned.
343	* @param pVCpu The cross context virtual CPU structure.
344	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
345	* out-of-sync. Make sure to update the required
346	* fields before using them.
347	* @param pVmxTransient Pointer to the VMX-transient structure.
348	*
349	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350	* use of that status code will be replaced with VINF_EM_SOMETHING
351	* later when switching over to IEM.
352	*/
353	#ifndef HMVMX_USE_FUNCTION_TABLE
354	typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355	#else
356	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357	#endif
358
359
360	/*********************************************************************************************************************************
361	* Internal Functions *
362	*********************************************************************************************************************************/
363	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368	bool fStepping, uint32_t *puIntState);
369	#if HC_ARCH_BITS == 32
370	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371	#endif
372	#ifndef HMVMX_USE_FUNCTION_TABLE
373	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376	#else
377	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379	#endif
380
381
382	/** @name VM-exit handlers.
383	* @{
384	*/
385	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397	static FNVMXEXITHANDLER hmR0VmxExitHlt;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413	static FNVMXEXITHANDLER hmR0VmxExitMwait;
414	static FNVMXEXITHANDLER hmR0VmxExitMtf;
415	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416	static FNVMXEXITHANDLER hmR0VmxExitPause;
417	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430	/** @} */
431
432	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443	/*********************************************************************************************************************************
444	* Global Variables *
445	*********************************************************************************************************************************/
446	#ifdef HMVMX_USE_FUNCTION_TABLE
447
448	/**
449	* VMX_EXIT dispatch table.
450	*/
451	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452	{
453	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493	/* 40 UNDEFINED */ hmR0VmxExitPause,
494	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513	/* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
515	/* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518	};
519	#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521	#ifdef VBOX_STRICT
522	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523	{
524	/* 0 */ "(Not Used)",
525	/* 1 */ "VMCALL executed in VMX root operation.",
526	/* 2 */ "VMCLEAR with invalid physical address.",
527	/* 3 */ "VMCLEAR with VMXON pointer.",
528	/* 4 */ "VMLAUNCH with non-clear VMCS.",
529	/* 5 */ "VMRESUME with non-launched VMCS.",
530	/* 6 */ "VMRESUME after VMXOFF",
531	/* 7 */ "VM-entry with invalid control fields.",
532	/* 8 */ "VM-entry with invalid host state fields.",
533	/* 9 */ "VMPTRLD with invalid physical address.",
534	/* 10 */ "VMPTRLD with VMXON pointer.",
535	/* 11 */ "VMPTRLD with incorrect revision identifier.",
536	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537	/* 13 */ "VMWRITE to read-only VMCS component.",
538	/* 14 */ "(Not Used)",
539	/* 15 */ "VMXON executed in VMX root operation.",
540	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541	/* 17 */ "VM-entry with non-launched executing VMCS.",
542	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543	/* 19 */ "VMCALL with non-clear VMCS.",
544	/* 20 */ "VMCALL with invalid VM-exit control fields.",
545	/* 21 */ "(Not Used)",
546	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548	/* 24 */ "VMCALL with invalid SMM-monitor features.",
549	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550	/* 26 */ "VM-entry with events blocked by MOV SS.",
551	/* 27 */ "(Not Used)",
552	/* 28 */ "Invalid operand to INVEPT/INVVPID."
553	};
554	#endif /* VBOX_STRICT */
555
556
557
558	/**
559	* Updates the VM's last error record.
560	*
561	* If there was a VMX instruction error, reads the error data from the VMCS and
562	* updates VCPU's last error record as well.
563	*
564	* @param pVM The cross context VM structure.
565	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
566	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567	* VERR_VMX_INVALID_VMCS_FIELD.
568	* @param rc The error code.
569	*/
570	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571	{
572	AssertPtr(pVM);
573	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
575	{
576	AssertPtrReturnVoid(pVCpu);
577	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578	}
579	pVM->hm.s.lLastError = rc;
580	}
581
582
583	/**
584	* Reads the VM-entry interruption-information field from the VMCS into the VMX
585	* transient structure.
586	*
587	* @returns VBox status code.
588	* @param pVmxTransient Pointer to the VMX transient structure.
589	*
590	* @remarks No-long-jump zone!!!
591	*/
592	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593	{
594	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595	AssertRCReturn(rc, rc);
596	return VINF_SUCCESS;
597	}
598
599
600	/**
601	* Reads the VM-entry exception error code field from the VMCS into
602	* the VMX transient structure.
603	*
604	* @returns VBox status code.
605	* @param pVmxTransient Pointer to the VMX transient structure.
606	*
607	* @remarks No-long-jump zone!!!
608	*/
609	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612	AssertRCReturn(rc, rc);
613	return VINF_SUCCESS;
614	}
615
616
617	/**
618	* Reads the VM-entry exception error code field from the VMCS into
619	* the VMX transient structure.
620	*
621	* @returns VBox status code.
622	* @param pVmxTransient Pointer to the VMX transient structure.
623	*
624	* @remarks No-long-jump zone!!!
625	*/
626	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627	{
628	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629	AssertRCReturn(rc, rc);
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit interruption-information field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVmxTransient Pointer to the VMX transient structure.
640	*/
641	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642	{
643	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644	{
645	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646	AssertRCReturn(rc, rc);
647	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648	}
649	return VINF_SUCCESS;
650	}
651
652
653	/**
654	* Reads the VM-exit interruption error code from the VMCS into the VMX
655	* transient structure.
656	*
657	* @returns VBox status code.
658	* @param pVmxTransient Pointer to the VMX transient structure.
659	*/
660	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661	{
662	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665	AssertRCReturn(rc, rc);
666	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667	}
668	return VINF_SUCCESS;
669	}
670
671
672	/**
673	* Reads the VM-exit instruction length field from the VMCS into the VMX
674	* transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVmxTransient Pointer to the VMX transient structure.
678	*/
679	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680	{
681	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682	{
683	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684	AssertRCReturn(rc, rc);
685	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686	}
687	return VINF_SUCCESS;
688	}
689
690
691	/**
692	* Reads the VM-exit instruction-information field from the VMCS into
693	* the VMX transient structure.
694	*
695	* @returns VBox status code.
696	* @param pVmxTransient Pointer to the VMX transient structure.
697	*/
698	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699	{
700	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701	{
702	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703	AssertRCReturn(rc, rc);
704	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705	}
706	return VINF_SUCCESS;
707	}
708
709
710	/**
711	* Reads the exit code qualification from the VMCS into the VMX transient
712	* structure.
713	*
714	* @returns VBox status code.
715	* @param pVCpu The cross context virtual CPU structure of the
716	* calling EMT. (Required for the VMCS cache case.)
717	* @param pVmxTransient Pointer to the VMX transient structure.
718	*/
719	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720	{
721	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722	{
723	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724	AssertRCReturn(rc, rc);
725	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726	}
727	return VINF_SUCCESS;
728	}
729
730
731	/**
732	* Reads the IDT-vectoring information field from the VMCS into the VMX
733	* transient structure.
734	*
735	* @returns VBox status code.
736	* @param pVmxTransient Pointer to the VMX transient structure.
737	*
738	* @remarks No-long-jump zone!!!
739	*/
740	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741	{
742	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743	{
744	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745	AssertRCReturn(rc, rc);
746	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747	}
748	return VINF_SUCCESS;
749	}
750
751
752	/**
753	* Reads the IDT-vectoring error code from the VMCS into the VMX
754	* transient structure.
755	*
756	* @returns VBox status code.
757	* @param pVmxTransient Pointer to the VMX transient structure.
758	*/
759	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760	{
761	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762	{
763	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764	AssertRCReturn(rc, rc);
765	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766	}
767	return VINF_SUCCESS;
768	}
769
770
771	/**
772	* Enters VMX root mode operation on the current CPU.
773	*
774	* @returns VBox status code.
775	* @param pVM The cross context VM structure. Can be
776	* NULL, after a resume.
777	* @param HCPhysCpuPage Physical address of the VMXON region.
778	* @param pvCpuPage Pointer to the VMXON region.
779	*/
780	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781	{
782	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784	Assert(pvCpuPage);
785	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787	if (pVM)
788	{
789	/* Write the VMCS revision dword to the VMXON region. */
790	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791	}
792
793	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794	RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796	/* Enable the VMX bit in CR4 if necessary. */
797	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799	/* Enter VMX root mode. */
800	int rc = VMXEnable(HCPhysCpuPage);
801	if (RT_FAILURE(rc))
802	{
803	if (!(uOldCr4 & X86_CR4_VMXE))
804	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806	if (pVM)
807	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808	}
809
810	/* Restore interrupts. */
811	ASMSetFlags(fEFlags);
812	return rc;
813	}
814
815
816	/**
817	* Exits VMX root mode operation on the current CPU.
818	*
819	* @returns VBox status code.
820	*/
821	static int hmR0VmxLeaveRootMode(void)
822	{
823	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826	RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828	/* If we're for some reason not in VMX root mode, then don't leave it. */
829	RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831	int rc;
832	if (uHostCR4 & X86_CR4_VMXE)
833	{
834	/* Exit VMX root mode and clear the VMX bit in CR4. */
835	VMXDisable();
836	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837	rc = VINF_SUCCESS;
838	}
839	else
840	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842	/* Restore interrupts. */
843	ASMSetFlags(fEFlags);
844	return rc;
845	}
846
847
848	/**
849	* Allocates and maps one physically contiguous page. The allocated page is
850	* zero'd out. (Used by various VT-x structures).
851	*
852	* @returns IPRT status code.
853	* @param pMemObj Pointer to the ring-0 memory object.
854	* @param ppVirt Where to store the virtual address of the
855	* allocation.
856	* @param pHCPhys Where to store the physical address of the
857	* allocation.
858	*/
859	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860	{
861	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866	if (RT_FAILURE(rc))
867	return rc;
868	ppVirt = RTR0MemObjAddress(pMemObj);
869	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
870	ASMMemZero32(*ppVirt, PAGE_SIZE);
871	return VINF_SUCCESS;
872	}
873
874
875	/**
876	* Frees and unmaps an allocated physical page.
877	*
878	* @param pMemObj Pointer to the ring-0 memory object.
879	* @param ppVirt Where to re-initialize the virtual address of
880	* allocation as 0.
881	* @param pHCPhys Where to re-initialize the physical address of the
882	* allocation as 0.
883	*/
884	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885	{
886	AssertPtr(pMemObj);
887	AssertPtr(ppVirt);
888	AssertPtr(pHCPhys);
889	if (*pMemObj != NIL_RTR0MEMOBJ)
890	{
891	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
892	AssertRC(rc);
893	*pMemObj = NIL_RTR0MEMOBJ;
894	*ppVirt = 0;
895	*pHCPhys = 0;
896	}
897	}
898
899
900	/**
901	* Worker function to free VT-x related structures.
902	*
903	* @returns IPRT status code.
904	* @param pVM The cross context VM structure.
905	*/
906	static void hmR0VmxStructsFree(PVM pVM)
907	{
908	for (VMCPUID i = 0; i < pVM->cCpus; i++)
909	{
910	PVMCPU pVCpu = &pVM->aCpus[i];
911	AssertPtr(pVCpu);
912
913	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921	}
922
923	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926	#endif
927	}
928
929
930	/**
931	* Worker function to allocate VT-x related VM structures.
932	*
933	* @returns IPRT status code.
934	* @param pVM The cross context VM structure.
935	*/
936	static int hmR0VmxStructsAlloc(PVM pVM)
937	{
938	/*
939	* Initialize members up-front so we can cleanup properly on allocation failure.
940	*/
941	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944	pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953	#endif
954	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957	for (VMCPUID i = 0; i < pVM->cCpus; i++)
958	{
959	PVMCPU pVCpu = &pVM->aCpus[i];
960	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965	}
966	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967	#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974	/*
975	* Allocate all the VT-x structures.
976	*/
977	int rc = VINF_SUCCESS;
978	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980	if (RT_FAILURE(rc))
981	goto cleanup;
982	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984	#endif
985
986	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988	{
989	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990	&pVM->hm.s.vmx.HCPhysApicAccess);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	/*
996	* Initialize per-VCPU VT-x structures.
997	*/
998	for (VMCPUID i = 0; i < pVM->cCpus; i++)
999	{
1000	PVMCPU pVCpu = &pVM->aCpus[i];
1001	AssertPtr(pVCpu);
1002
1003	/* Allocate the VM control structure (VMCS). */
1004	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005	if (RT_FAILURE(rc))
1006	goto cleanup;
1007
1008	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010	{
1011	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012	&pVCpu->hm.s.vmx.HCPhysVirtApic);
1013	if (RT_FAILURE(rc))
1014	goto cleanup;
1015	}
1016
1017	/*
1018	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019	* transparent accesses of specific MSRs.
1020	*
1021	* If the condition for enabling MSR bitmaps changes here, don't forget to
1022	* update HMAreMsrBitmapsAvailable().
1023	*/
1024	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025	{
1026	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028	if (RT_FAILURE(rc))
1029	goto cleanup;
1030	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031	}
1032
1033	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035	if (RT_FAILURE(rc))
1036	goto cleanup;
1037
1038	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1039	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040	if (RT_FAILURE(rc))
1041	goto cleanup;
1042	}
1043
1044	return VINF_SUCCESS;
1045
1046	cleanup:
1047	hmR0VmxStructsFree(pVM);
1048	return rc;
1049	}
1050
1051
1052	/**
1053	* Does global VT-x initialization (called during module initialization).
1054	*
1055	* @returns VBox status code.
1056	*/
1057	VMMR0DECL(int) VMXR0GlobalInit(void)
1058	{
1059	#ifdef HMVMX_USE_FUNCTION_TABLE
1060	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061	# ifdef VBOX_STRICT
1062	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063	Assert(g_apfnVMExitHandlers[i]);
1064	# endif
1065	#endif
1066	return VINF_SUCCESS;
1067	}
1068
1069
1070	/**
1071	* Does global VT-x termination (called during module termination).
1072	*/
1073	VMMR0DECL(void) VMXR0GlobalTerm()
1074	{
1075	/* Nothing to do currently. */
1076	}
1077
1078
1079	/**
1080	* Sets up and activates VT-x on the current CPU.
1081	*
1082	* @returns VBox status code.
1083	* @param pCpu Pointer to the global CPU info struct.
1084	* @param pVM The cross context VM structure. Can be
1085	* NULL after a host resume operation.
1086	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087	* fEnabledByHost is @c true).
1088	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089	* @a fEnabledByHost is @c true).
1090	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091	* enable VT-x on the host.
1092	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1093	*/
1094	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095	void *pvMsrs)
1096	{
1097	Assert(pCpu);
1098	Assert(pvMsrs);
1099	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101	/* Enable VT-x if it's not already enabled by the host. */
1102	if (!fEnabledByHost)
1103	{
1104	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105	if (RT_FAILURE(rc))
1106	return rc;
1107	}
1108
1109	/*
1110	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112	*/
1113	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115	{
1116	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117	pCpu->fFlushAsidBeforeUse = false;
1118	}
1119	else
1120	pCpu->fFlushAsidBeforeUse = true;
1121
1122	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123	++pCpu->cTlbFlushes;
1124
1125	return VINF_SUCCESS;
1126	}
1127
1128
1129	/**
1130	* Deactivates VT-x on the current CPU.
1131	*
1132	* @returns VBox status code.
1133	* @param pCpu Pointer to the global CPU info struct.
1134	* @param pvCpuPage Pointer to the VMXON region.
1135	* @param HCPhysCpuPage Physical address of the VMXON region.
1136	*
1137	* @remarks This function should never be called when SUPR0EnableVTx() or
1138	* similar was used to enable VT-x on the host.
1139	*/
1140	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141	{
1142	NOREF(pCpu);
1143	NOREF(pvCpuPage);
1144	NOREF(HCPhysCpuPage);
1145
1146	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147	return hmR0VmxLeaveRootMode();
1148	}
1149
1150
1151	/**
1152	* Sets the permission bits for the specified MSR in the MSR bitmap.
1153	*
1154	* @param pVCpu The cross context virtual CPU structure.
1155	* @param uMsr The MSR value.
1156	* @param enmRead Whether reading this MSR causes a VM-exit.
1157	* @param enmWrite Whether writing this MSR causes a VM-exit.
1158	*/
1159	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160	{
1161	int32_t iBit;
1162	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164	/*
1165	* Layout:
1166	* 0x000 - 0x3ff - Low MSR read bits
1167	* 0x400 - 0x7ff - High MSR read bits
1168	* 0x800 - 0xbff - Low MSR write bits
1169	* 0xc00 - 0xfff - High MSR write bits
1170	*/
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174	{
1175	iBit = uMsr - UINT32_C(0xC0000000);
1176	pbMsrBitmap += 0x400;
1177	}
1178	else
1179	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181	Assert(iBit <= 0x1fff);
1182	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183	ASMBitSet(pbMsrBitmap, iBit);
1184	else
1185	ASMBitClear(pbMsrBitmap, iBit);
1186
1187	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189	else
1190	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191	}
1192
1193
1194	#ifdef VBOX_STRICT
1195	/**
1196	* Gets the permission bits for the specified MSR in the MSR bitmap.
1197	*
1198	* @returns VBox status code.
1199	* @retval VINF_SUCCESS if the specified MSR is found.
1200	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1201	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202	*
1203	* @param pVCpu The cross context virtual CPU structure.
1204	* @param uMsr The MSR.
1205	* @param penmRead Where to store the read permissions.
1206	* @param penmWrite Where to store the write permissions.
1207	*/
1208	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209	{
1210	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212	int32_t iBit;
1213	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215	/* See hmR0VmxSetMsrPermission() for the layout. */
1216	if (uMsr <= 0x00001FFF)
1217	iBit = uMsr;
1218	else if ( uMsr >= 0xC0000000
1219	&& uMsr <= 0xC0001FFF)
1220	{
1221	iBit = (uMsr - 0xC0000000);
1222	pbMsrBitmap += 0x400;
1223	}
1224	else
1225	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227	Assert(iBit <= 0x1fff);
1228	if (ASMBitTest(pbMsrBitmap, iBit))
1229	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1230	else
1231	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235	else
1236	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237	return VINF_SUCCESS;
1238	}
1239	#endif /* VBOX_STRICT */
1240
1241
1242	/**
1243	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244	* area.
1245	*
1246	* @returns VBox status code.
1247	* @param pVCpu The cross context virtual CPU structure.
1248	* @param cMsrs The number of MSRs.
1249	*/
1250	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251	{
1252	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255	{
1256	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259	}
1260
1261	/* Update number of guest MSRs to load/store across the world-switch. */
1262	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267	AssertRCReturn(rc, rc);
1268
1269	/* Update the VCPU's copy of the MSR count. */
1270	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272	return VINF_SUCCESS;
1273	}
1274
1275
1276	/**
1277	* Adds a new (or updates the value of an existing) guest/host MSR
1278	* pair to be swapped during the world-switch as part of the
1279	* auto-load/store MSR area in the VMCS.
1280	*
1281	* @returns VBox status code.
1282	* @param pVCpu The cross context virtual CPU structure.
1283	* @param uMsr The MSR.
1284	* @param uGuestMsrValue Value of the guest MSR.
1285	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1286	* necessary.
1287	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288	* its value was updated. Optional, can be NULL.
1289	*/
1290	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291	bool *pfAddedAndUpdated)
1292	{
1293	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295	uint32_t i;
1296	for (i = 0; i < cMsrs; i++)
1297	{
1298	if (pGuestMsr->u32Msr == uMsr)
1299	break;
1300	pGuestMsr++;
1301	}
1302
1303	bool fAdded = false;
1304	if (i == cMsrs)
1305	{
1306	++cMsrs;
1307	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314	fAdded = true;
1315	}
1316
1317	/* Update the MSR values in the auto-load/store MSR area. */
1318	pGuestMsr->u32Msr = uMsr;
1319	pGuestMsr->u64Value = uGuestMsrValue;
1320
1321	/* Create/update the MSR slot in the host MSR area. */
1322	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323	pHostMsr += i;
1324	pHostMsr->u32Msr = uMsr;
1325
1326	/*
1327	* Update the host MSR only when requested by the caller AND when we're
1328	* adding it to the auto-load/store area. Otherwise, it would have been
1329	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330	*/
1331	bool fUpdatedMsrValue = false;
1332	if ( fAdded
1333	&& fUpdateHostMsr)
1334	{
1335	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338	fUpdatedMsrValue = true;
1339	}
1340
1341	if (pfAddedAndUpdated)
1342	*pfAddedAndUpdated = fUpdatedMsrValue;
1343	return VINF_SUCCESS;
1344	}
1345
1346
1347	/**
1348	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1349	* auto-load/store MSR area in the VMCS.
1350	*
1351	* @returns VBox status code.
1352	* @param pVCpu The cross context virtual CPU structure.
1353	* @param uMsr The MSR.
1354	*/
1355	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356	{
1357	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359	for (uint32_t i = 0; i < cMsrs; i++)
1360	{
1361	/* Find the MSR. */
1362	if (pGuestMsr->u32Msr == uMsr)
1363	{
1364	/* If it's the last MSR, simply reduce the count. */
1365	if (i == cMsrs - 1)
1366	{
1367	--cMsrs;
1368	break;
1369	}
1370
1371	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373	pLastGuestMsr += cMsrs - 1;
1374	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379	pLastHostMsr += cMsrs - 1;
1380	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381	pHostMsr->u64Value = pLastHostMsr->u64Value;
1382	--cMsrs;
1383	break;
1384	}
1385	pGuestMsr++;
1386	}
1387
1388	/* Update the VMCS if the count changed (meaning the MSR was found). */
1389	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390	{
1391	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392	AssertRCReturn(rc, rc);
1393
1394	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399	return VINF_SUCCESS;
1400	}
1401
1402	return VERR_NOT_FOUND;
1403	}
1404
1405
1406	/**
1407	* Checks if the specified guest MSR is part of the auto-load/store area in
1408	* the VMCS.
1409	*
1410	* @returns true if found, false otherwise.
1411	* @param pVCpu The cross context virtual CPU structure.
1412	* @param uMsr The MSR to find.
1413	*/
1414	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415	{
1416	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420	{
1421	if (pGuestMsr->u32Msr == uMsr)
1422	return true;
1423	}
1424	return false;
1425	}
1426
1427
1428	/**
1429	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430	*
1431	* @param pVCpu The cross context virtual CPU structure.
1432	*
1433	* @remarks No-long-jump zone!!!
1434	*/
1435	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436	{
1437	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443	{
1444	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446	/*
1447	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449	*/
1450	if (pHostMsr->u32Msr == MSR_K6_EFER)
1451	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452	else
1453	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454	}
1455
1456	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457	}
1458
1459
1460	/**
1461	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1462	* perform lazy restoration of the host MSRs while leaving VT-x.
1463	*
1464	* @param pVCpu The cross context virtual CPU structure.
1465	*
1466	* @remarks No-long-jump zone!!!
1467	*/
1468	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469	{
1470	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472	/*
1473	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474	*/
1475	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST));
1476	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477	{
1478	#if HC_ARCH_BITS == 64
1479	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480	{
1481	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485	}
1486	#endif
1487	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1488	}
1489	}
1490
1491
1492	/**
1493	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1494	* lazily while leaving VT-x.
1495	*
1496	* @returns true if it does, false otherwise.
1497	* @param pVCpu The cross context virtual CPU structure.
1498	* @param uMsr The MSR to check.
1499	*/
1500	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501	{
1502	NOREF(pVCpu);
1503	#if HC_ARCH_BITS == 64
1504	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505	{
1506	switch (uMsr)
1507	{
1508	case MSR_K8_LSTAR:
1509	case MSR_K6_STAR:
1510	case MSR_K8_SF_MASK:
1511	case MSR_K8_KERNEL_GS_BASE:
1512	return true;
1513	}
1514	}
1515	#endif
1516	return false;
1517	}
1518
1519
1520	/**
1521	* Saves a set of guest MSRs back into the guest-CPU context.
1522	*
1523	* @param pVCpu The cross context virtual CPU structure.
1524	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525	* out-of-sync. Make sure to update the required fields
1526	* before using them.
1527	*
1528	* @remarks No-long-jump zone!!!
1529	*/
1530	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531	{
1532	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536	{
1537	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538	#if HC_ARCH_BITS == 64
1539	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540	{
1541	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545	}
1546	#endif
1547	}
1548	}
1549
1550
1551	/**
1552	* Loads a set of guests MSRs to allow read/passthru to the guest.
1553	*
1554	* The name of this function is slightly confusing. This function does NOT
1555	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556	* common prefix for functions dealing with "lazy restoration" of the shared
1557	* MSRs.
1558	*
1559	* @param pVCpu The cross context virtual CPU structure.
1560	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561	* out-of-sync. Make sure to update the required fields
1562	* before using them.
1563	*
1564	* @remarks No-long-jump zone!!!
1565	*/
1566	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567	{
1568	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572	do { \
1573	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575	else \
1576	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577	} while (0)
1578
1579	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581	{
1582	#if HC_ARCH_BITS == 64
1583	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584	{
1585	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589	}
1590	#endif
1591	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1592	}
1593
1594	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595	}
1596
1597
1598	/**
1599	* Performs lazy restoration of the set of host MSRs if they were previously
1600	* loaded with guest MSR values.
1601	*
1602	* @param pVCpu The cross context virtual CPU structure.
1603	*
1604	* @remarks No-long-jump zone!!!
1605	* @remarks The guest MSRs should have been saved back into the guest-CPU
1606	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1607	*/
1608	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609	{
1610	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614	{
1615	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616	#if HC_ARCH_BITS == 64
1617	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618	{
1619	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623	}
1624	#endif
1625	}
1626	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1627	}
1628
1629
1630	/**
1631	* Verifies that our cached values of the VMCS controls are all
1632	* consistent with what's actually present in the VMCS.
1633	*
1634	* @returns VBox status code.
1635	* @param pVCpu The cross context virtual CPU structure.
1636	*/
1637	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638	{
1639	uint32_t u32Val;
1640	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641	AssertRCReturn(rc, rc);
1642	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646	AssertRCReturn(rc, rc);
1647	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651	AssertRCReturn(rc, rc);
1652	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656	AssertRCReturn(rc, rc);
1657	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661	{
1662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663	AssertRCReturn(rc, rc);
1664	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667	}
1668
1669	return VINF_SUCCESS;
1670	}
1671
1672
1673	#ifdef VBOX_STRICT
1674	/**
1675	* Verifies that our cached host EFER value has not changed
1676	* since we cached it.
1677	*
1678	* @param pVCpu The cross context virtual CPU structure.
1679	*/
1680	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681	{
1682	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685	{
1686	uint64_t u64Val;
1687	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688	AssertRC(rc);
1689
1690	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692	}
1693	}
1694
1695
1696	/**
1697	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698	* VMCS are correct.
1699	*
1700	* @param pVCpu The cross context virtual CPU structure.
1701	*/
1702	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703	{
1704	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706	/* Verify MSR counts in the VMCS are what we think it should be. */
1707	uint32_t cMsrs;
1708	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720	{
1721	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723	pGuestMsr->u32Msr, cMsrs));
1724
1725	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729	/* Verify that the permissions are as expected in the MSR bitmap. */
1730	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731	{
1732	VMXMSREXITREAD enmRead;
1733	VMXMSREXITWRITE enmWrite;
1734	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737	{
1738	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740	}
1741	else
1742	{
1743	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744	pGuestMsr->u32Msr, cMsrs));
1745	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746	pGuestMsr->u32Msr, cMsrs));
1747	}
1748	}
1749	}
1750	}
1751	#endif /* VBOX_STRICT */
1752
1753
1754	/**
1755	* Flushes the TLB using EPT.
1756	*
1757	* @returns VBox status code.
1758	* @param pVCpu The cross context virtual CPU structure of the calling
1759	* EMT. Can be NULL depending on @a enmFlush.
1760	* @param enmFlush Type of flush.
1761	*
1762	* @remarks Caller is responsible for making sure this function is called only
1763	* when NestedPaging is supported and providing @a enmFlush that is
1764	* supported by the CPU.
1765	* @remarks Can be called with interrupts disabled.
1766	*/
1767	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768	{
1769	uint64_t au64Descriptor[2];
1770	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771	au64Descriptor[0] = 0;
1772	else
1773	{
1774	Assert(pVCpu);
1775	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776	}
1777	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781	rc));
1782	if ( RT_SUCCESS(rc)
1783	&& pVCpu)
1784	{
1785	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786	}
1787	}
1788
1789
1790	/**
1791	* Flushes the TLB using VPID.
1792	*
1793	* @returns VBox status code.
1794	* @param pVM The cross context VM structure.
1795	* @param pVCpu The cross context virtual CPU structure of the calling
1796	* EMT. Can be NULL depending on @a enmFlush.
1797	* @param enmFlush Type of flush.
1798	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1799	* on @a enmFlush).
1800	*
1801	* @remarks Can be called with interrupts disabled.
1802	*/
1803	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804	{
1805	NOREF(pVM);
1806	AssertPtr(pVM);
1807	Assert(pVM->hm.s.vmx.fVpid);
1808
1809	uint64_t au64Descriptor[2];
1810	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811	{
1812	au64Descriptor[0] = 0;
1813	au64Descriptor[1] = 0;
1814	}
1815	else
1816	{
1817	AssertPtr(pVCpu);
1818	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821	au64Descriptor[1] = GCPtr;
1822	}
1823
1824	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825	AssertMsg(rc == VINF_SUCCESS,
1826	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827	if ( RT_SUCCESS(rc)
1828	&& pVCpu)
1829	{
1830	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831	}
1832	}
1833
1834
1835	/**
1836	* Invalidates a guest page by guest virtual address. Only relevant for
1837	* EPT/VPID, otherwise there is nothing really to invalidate.
1838	*
1839	* @returns VBox status code.
1840	* @param pVM The cross context VM structure.
1841	* @param pVCpu The cross context virtual CPU structure.
1842	* @param GCVirt Guest virtual address of the page to invalidate.
1843	*/
1844	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845	{
1846	AssertPtr(pVM);
1847	AssertPtr(pVCpu);
1848	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851	if (!fFlushPending)
1852	{
1853	/*
1854	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855	* See @bugref{6043} and @bugref{6177}.
1856	*
1857	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858	* function maybe called in a loop with individual addresses.
1859	*/
1860	if (pVM->hm.s.vmx.fVpid)
1861	{
1862	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863	{
1864	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866	}
1867	else
1868	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869	}
1870	else if (pVM->hm.s.fNestedPaging)
1871	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872	}
1873
1874	return VINF_SUCCESS;
1875	}
1876
1877
1878	/**
1879	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880	* otherwise there is nothing really to invalidate.
1881	*
1882	* @returns VBox status code.
1883	* @param pVM The cross context VM structure.
1884	* @param pVCpu The cross context virtual CPU structure.
1885	* @param GCPhys Guest physical address of the page to invalidate.
1886	*/
1887	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888	{
1889	NOREF(pVM); NOREF(GCPhys);
1890	LogFlowFunc(("%RGp\n", GCPhys));
1891
1892	/*
1893	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896	*/
1897	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899	return VINF_SUCCESS;
1900	}
1901
1902
1903	/**
1904	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905	* case where neither EPT nor VPID is supported by the CPU.
1906	*
1907	* @param pVM The cross context VM structure.
1908	* @param pVCpu The cross context virtual CPU structure.
1909	* @param pCpu Pointer to the global HM struct.
1910	*
1911	* @remarks Called with interrupts disabled.
1912	*/
1913	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914	{
1915	AssertPtr(pVCpu);
1916	AssertPtr(pCpu);
1917	NOREF(pVM);
1918
1919	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921	Assert(pCpu->idCpu != NIL_RTCPUID);
1922	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924	pVCpu->hm.s.fForceTLBFlush = false;
1925	return;
1926	}
1927
1928
1929	/**
1930	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931	*
1932	* @param pVM The cross context VM structure.
1933	* @param pVCpu The cross context virtual CPU structure.
1934	* @param pCpu Pointer to the global HM CPU struct.
1935	* @remarks All references to "ASID" in this function pertains to "VPID" in
1936	* Intel's nomenclature. The reason is, to avoid confusion in compare
1937	* statements since the host-CPU copies are named "ASID".
1938	*
1939	* @remarks Called with interrupts disabled.
1940	*/
1941	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942	{
1943	#ifdef VBOX_WITH_STATISTICS
1944	bool fTlbFlushed = false;
1945	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947	if (!fTlbFlushed) \
1948	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949	} while (0)
1950	#else
1951	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953	#endif
1954
1955	AssertPtr(pVM);
1956	AssertPtr(pCpu);
1957	AssertPtr(pVCpu);
1958	Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964	/*
1965	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968	*/
1969	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971	{
1972	++pCpu->uCurrentAsid;
1973	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974	{
1975	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978	}
1979
1980	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984	/*
1985	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987	*/
1988	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990	HMVMX_SET_TAGGED_TLB_FLUSHED();
1991	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992	}
1993
1994	/* Check for explicit TLB flushes. */
1995	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996	{
1997	/*
1998	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001	* but not guest-physical mappings.
2002	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003	*/
2004	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006	HMVMX_SET_TAGGED_TLB_FLUSHED();
2007	}
2008
2009	pVCpu->hm.s.fForceTLBFlush = false;
2010	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022	/* Update VMCS with the VPID. */
2023	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024	AssertRC(rc);
2025
2026	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027	}
2028
2029
2030	/**
2031	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032	*
2033	* @returns VBox status code.
2034	* @param pVM The cross context VM structure.
2035	* @param pVCpu The cross context virtual CPU structure.
2036	* @param pCpu Pointer to the global HM CPU struct.
2037	*
2038	* @remarks Called with interrupts disabled.
2039	*/
2040	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041	{
2042	AssertPtr(pVM);
2043	AssertPtr(pVCpu);
2044	AssertPtr(pCpu);
2045	Assert(pCpu->idCpu != NIL_RTCPUID);
2046	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049	/*
2050	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052	*/
2053	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055	{
2056	pVCpu->hm.s.fForceTLBFlush = true;
2057	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058	}
2059
2060	/* Check for explicit TLB flushes. */
2061	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062	{
2063	pVCpu->hm.s.fForceTLBFlush = true;
2064	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065	}
2066
2067	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070	if (pVCpu->hm.s.fForceTLBFlush)
2071	{
2072	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073	pVCpu->hm.s.fForceTLBFlush = false;
2074	}
2075	}
2076
2077
2078	/**
2079	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080	*
2081	* @returns VBox status code.
2082	* @param pVM The cross context VM structure.
2083	* @param pVCpu The cross context virtual CPU structure.
2084	* @param pCpu Pointer to the global HM CPU struct.
2085	*
2086	* @remarks Called with interrupts disabled.
2087	*/
2088	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089	{
2090	AssertPtr(pVM);
2091	AssertPtr(pVCpu);
2092	AssertPtr(pCpu);
2093	Assert(pCpu->idCpu != NIL_RTCPUID);
2094	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097	/*
2098	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101	*/
2102	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104	{
2105	pVCpu->hm.s.fForceTLBFlush = true;
2106	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107	}
2108
2109	/* Check for explicit TLB flushes. */
2110	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111	{
2112	/*
2113	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116	*/
2117	pVCpu->hm.s.fForceTLBFlush = true;
2118	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119	}
2120
2121	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122	if (pVCpu->hm.s.fForceTLBFlush)
2123	{
2124	++pCpu->uCurrentAsid;
2125	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126	{
2127	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130	}
2131
2132	pVCpu->hm.s.fForceTLBFlush = false;
2133	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135	if (pCpu->fFlushAsidBeforeUse)
2136	{
2137	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140	{
2141	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142	pCpu->fFlushAsidBeforeUse = false;
2143	}
2144	else
2145	{
2146	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148	}
2149	}
2150	}
2151
2152	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161	AssertRC(rc);
2162	}
2163
2164
2165	/**
2166	* Flushes the guest TLB entry based on CPU capabilities.
2167	*
2168	* @param pVCpu The cross context virtual CPU structure.
2169	* @param pCpu Pointer to the global HM CPU struct.
2170	*/
2171	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172	{
2173	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175	#endif
2176	PVM pVM = pVCpu->CTX_SUFF(pVM);
2177	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178	{
2179	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183	default:
2184	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185	break;
2186	}
2187
2188	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189	}
2190
2191
2192	/**
2193	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194	* TLB entries from the host TLB before VM-entry.
2195	*
2196	* @returns VBox status code.
2197	* @param pVM The cross context VM structure.
2198	*/
2199	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200	{
2201	/*
2202	* Determine optimal flush type for Nested Paging.
2203	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204	* guest execution (see hmR3InitFinalizeR0()).
2205	*/
2206	if (pVM->hm.s.fNestedPaging)
2207	{
2208	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209	{
2210	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214	else
2215	{
2216	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220	}
2221
2222	/* Make sure the write-back cacheable memory type for EPT is supported. */
2223	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224	{
2225	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228	}
2229
2230	/* EPT requires a page-walk length of 4. */
2231	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232	{
2233	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236	}
2237	}
2238	else
2239	{
2240	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244	}
2245	}
2246
2247	/*
2248	* Determine optimal flush type for VPID.
2249	*/
2250	if (pVM->hm.s.vmx.fVpid)
2251	{
2252	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253	{
2254	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258	else
2259	{
2260	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266	pVM->hm.s.vmx.fVpid = false;
2267	}
2268	}
2269	else
2270	{
2271	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274	pVM->hm.s.vmx.fVpid = false;
2275	}
2276	}
2277
2278	/*
2279	* Setup the handler for flushing tagged-TLBs.
2280	*/
2281	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283	else if (pVM->hm.s.fNestedPaging)
2284	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285	else if (pVM->hm.s.vmx.fVpid)
2286	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287	else
2288	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289	return VINF_SUCCESS;
2290	}
2291
2292
2293	/**
2294	* Sets up pin-based VM-execution controls in the VMCS.
2295	*
2296	* @returns VBox status code.
2297	* @param pVM The cross context VM structure.
2298	* @param pVCpu The cross context virtual CPU structure.
2299	*/
2300	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301	{
2302	AssertPtr(pVM);
2303	AssertPtr(pVCpu);
2304
2305	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314	/* Enable the VMX preemption timer. */
2315	if (pVM->hm.s.vmx.fUsePreemptTimer)
2316	{
2317	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319	}
2320
2321	#ifdef VBOX_WITH_NEW_APIC
2322	#if 0
2323	/* Enable posted-interrupt processing. */
2324	if (pVM->hm.s.fPostedIntrs)
2325	{
2326	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329	}
2330	#endif
2331	#endif
2332
2333	if ((val & zap) != val)
2334	{
2335	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339	}
2340
2341	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342	AssertRCReturn(rc, rc);
2343
2344	pVCpu->hm.s.vmx.u32PinCtls = val;
2345	return rc;
2346	}
2347
2348
2349	/**
2350	* Sets up processor-based VM-execution controls in the VMCS.
2351	*
2352	* @returns VBox status code.
2353	* @param pVM The cross context VM structure.
2354	* @param pVCpu The cross context virtual CPU structure.
2355	*/
2356	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357	{
2358	AssertPtr(pVM);
2359	AssertPtr(pVCpu);
2360
2361	int rc = VERR_INTERNAL_ERROR_5;
2362	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376	{
2377	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380	}
2381
2382	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383	if (!pVM->hm.s.fNestedPaging)
2384	{
2385	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389	}
2390
2391	/* Use TPR shadowing if supported by the CPU. */
2392	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393	{
2394	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398	AssertRCReturn(rc, rc);
2399
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401	/* CR8 writes cause a VM-exit based on TPR threshold. */
2402	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404	}
2405	else
2406	{
2407	/*
2408	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409	* Set this control only for 64-bit guests.
2410	*/
2411	if (pVM->hm.s.fAllow64BitGuests)
2412	{
2413	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415	}
2416	}
2417
2418	/* Use MSR-bitmaps if supported by the CPU. */
2419	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420	{
2421	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426	AssertRCReturn(rc, rc);
2427
2428	/*
2429	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430	* automatically using dedicated fields in the VMCS.
2431	*/
2432	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438	#if HC_ARCH_BITS == 64
2439	/*
2440	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441	*/
2442	if (pVM->hm.s.fAllow64BitGuests)
2443	{
2444	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448	}
2449	#endif
2450	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451	}
2452
2453	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457	if ((val & zap) != val)
2458	{
2459	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463	}
2464
2465	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466	AssertRCReturn(rc, rc);
2467
2468	pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470	/*
2471	* Secondary processor-based VM-execution controls.
2472	*/
2473	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474	{
2475	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481	if (pVM->hm.s.fNestedPaging)
2482	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483	else
2484	{
2485	/*
2486	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489	*/
2490	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492	}
2493
2494	if (pVM->hm.s.vmx.fVpid)
2495	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500	#ifdef VBOX_WITH_NEW_APIC
2501	#if 0
2502	if (pVM->hm.s.fVirtApicRegs)
2503	{
2504	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509	}
2510	#endif
2511	#endif
2512
2513	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515	* done dynamically. */
2516	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517	{
2518	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522	AssertRCReturn(rc, rc);
2523	}
2524
2525	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529	&& pVM->hm.s.vmx.cPleGapTicks
2530	&& pVM->hm.s.vmx.cPleWindowTicks)
2531	{
2532	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536	AssertRCReturn(rc, rc);
2537	}
2538
2539	if ((val & zap) != val)
2540	{
2541	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545	}
2546
2547	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548	AssertRCReturn(rc, rc);
2549
2550	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551	}
2552	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553	{
2554	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555	"available\n"));
2556	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558	}
2559
2560	return VINF_SUCCESS;
2561	}
2562
2563
2564	/**
2565	* Sets up miscellaneous (everything other than Pin & Processor-based
2566	* VM-execution) control fields in the VMCS.
2567	*
2568	* @returns VBox status code.
2569	* @param pVM The cross context VM structure.
2570	* @param pVCpu The cross context virtual CPU structure.
2571	*/
2572	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573	{
2574	NOREF(pVM);
2575	AssertPtr(pVM);
2576	AssertPtr(pVCpu);
2577
2578	int rc = VERR_GENERAL_FAILURE;
2579
2580	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581	#if 0
2582	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586	/*
2587	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589	* We thus use the exception bitmap to control it rather than use both.
2590	*/
2591	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594	/** @todo Explore possibility of using IO-bitmaps. */
2595	/* All IO & IOIO instructions cause VM-exits. */
2596	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599	/* Initialize the MSR-bitmap area. */
2600	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603	AssertRCReturn(rc, rc);
2604	#endif
2605
2606	/* Setup MSR auto-load/store area. */
2607	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611	AssertRCReturn(rc, rc);
2612
2613	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616	AssertRCReturn(rc, rc);
2617
2618	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620	AssertRCReturn(rc, rc);
2621
2622	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623	#if 0
2624	/* Setup debug controls */
2625	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627	AssertRCReturn(rc, rc);
2628	#endif
2629
2630	return rc;
2631	}
2632
2633
2634	/**
2635	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2636	*
2637	* @returns VBox status code.
2638	* @param pVM The cross context VM structure.
2639	* @param pVCpu The cross context virtual CPU structure.
2640	*/
2641	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642	{
2643	AssertPtr(pVM);
2644	AssertPtr(pVCpu);
2645
2646	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648	uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2652
2653	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2654	and writes, and because recursive #DBs can cause the CPU hang, we must always
2655	intercept #DB. */
2656	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2657
2658	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659	if (!pVM->hm.s.fNestedPaging)
2660	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2661
2662	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664	AssertRCReturn(rc, rc);
2665	return rc;
2666	}
2667
2668
2669	/**
2670	* Sets up the initial guest-state mask. The guest-state mask is consulted
2671	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2672	* for the nested virtualization case (as it would cause a VM-exit).
2673	*
2674	* @param pVCpu The cross context virtual CPU structure.
2675	*/
2676	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677	{
2678	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680	return VINF_SUCCESS;
2681	}
2682
2683
2684	/**
2685	* Does per-VM VT-x initialization.
2686	*
2687	* @returns VBox status code.
2688	* @param pVM The cross context VM structure.
2689	*/
2690	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691	{
2692	LogFlowFunc(("pVM=%p\n", pVM));
2693
2694	int rc = hmR0VmxStructsAlloc(pVM);
2695	if (RT_FAILURE(rc))
2696	{
2697	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698	return rc;
2699	}
2700
2701	return VINF_SUCCESS;
2702	}
2703
2704
2705	/**
2706	* Does per-VM VT-x termination.
2707	*
2708	* @returns VBox status code.
2709	* @param pVM The cross context VM structure.
2710	*/
2711	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712	{
2713	LogFlowFunc(("pVM=%p\n", pVM));
2714
2715	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718	#endif
2719	hmR0VmxStructsFree(pVM);
2720	return VINF_SUCCESS;
2721	}
2722
2723
2724	/**
2725	* Sets up the VM for execution under VT-x.
2726	* This function is only called once per-VM during initialization.
2727	*
2728	* @returns VBox status code.
2729	* @param pVM The cross context VM structure.
2730	*/
2731	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732	{
2733	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736	LogFlowFunc(("pVM=%p\n", pVM));
2737
2738	/*
2739	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2740	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741	*/
2742	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2745	{
2746	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747	return VERR_INTERNAL_ERROR;
2748	}
2749
2750	/* Initialize these always, see hmR3InitFinalizeR0().*/
2751	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754	/* Setup the tagged-TLB flush handlers. */
2755	int rc = hmR0VmxSetupTaggedTlb(pVM);
2756	if (RT_FAILURE(rc))
2757	{
2758	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759	return rc;
2760	}
2761
2762	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764	#if HC_ARCH_BITS == 64
2765	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768	{
2769	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770	}
2771	#endif
2772
2773	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774	RTCCUINTREG uHostCR4 = ASMGetCR4();
2775	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779	{
2780	PVMCPU pVCpu = &pVM->aCpus[i];
2781	AssertPtr(pVCpu);
2782	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2785	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791	/* Set revision dword at the beginning of the VMCS structure. */
2792	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799	/* Load this VMCS as the current VMCS. */
2800	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824	#if HC_ARCH_BITS == 32
2825	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828	#endif
2829
2830	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838	}
2839
2840	return VINF_SUCCESS;
2841	}
2842
2843
2844	/**
2845	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846	* the VMCS.
2847	*
2848	* @returns VBox status code.
2849	* @param pVM The cross context VM structure.
2850	* @param pVCpu The cross context virtual CPU structure.
2851	*/
2852	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853	{
2854	NOREF(pVM); NOREF(pVCpu);
2855
2856	RTCCUINTREG uReg = ASMGetCR0();
2857	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858	AssertRCReturn(rc, rc);
2859
2860	uReg = ASMGetCR3();
2861	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862	AssertRCReturn(rc, rc);
2863
2864	uReg = ASMGetCR4();
2865	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866	AssertRCReturn(rc, rc);
2867	return rc;
2868	}
2869
2870
2871	#if HC_ARCH_BITS == 64
2872	/**
2873	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874	* requirements. See hmR0VmxSaveHostSegmentRegs().
2875	*/
2876	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2878	{ \
2879	bool fValidSelector = true; \
2880	if ((selValue) & X86_SEL_LDT) \
2881	{ \
2882	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884	} \
2885	if (fValidSelector) \
2886	{ \
2887	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2888	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889	} \
2890	(selValue) = 0; \
2891	}
2892	#endif
2893
2894
2895	/**
2896	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897	* the host-state area in the VMCS.
2898	*
2899	* @returns VBox status code.
2900	* @param pVM The cross context VM structure.
2901	* @param pVCpu The cross context virtual CPU structure.
2902	*/
2903	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904	{
2905	int rc = VERR_INTERNAL_ERROR_5;
2906
2907	#if HC_ARCH_BITS == 64
2908	/*
2909	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911	*/
2912	AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2913	("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2914	#endif
2915
2916	/*
2917	* Host DS, ES, FS and GS segment registers.
2918	*/
2919	#if HC_ARCH_BITS == 64
2920	RTSEL uSelDS = ASMGetDS();
2921	RTSEL uSelES = ASMGetES();
2922	RTSEL uSelFS = ASMGetFS();
2923	RTSEL uSelGS = ASMGetGS();
2924	#else
2925	RTSEL uSelDS = 0;
2926	RTSEL uSelES = 0;
2927	RTSEL uSelFS = 0;
2928	RTSEL uSelGS = 0;
2929	#endif
2930
2931	/* Recalculate which host-state bits need to be manually restored. */
2932	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2933
2934	/*
2935	* Host CS and SS segment registers.
2936	*/
2937	RTSEL uSelCS = ASMGetCS();
2938	RTSEL uSelSS = ASMGetSS();
2939
2940	/*
2941	* Host TR segment register.
2942	*/
2943	RTSEL uSelTR = ASMGetTR();
2944
2945	#if HC_ARCH_BITS == 64
2946	/*
2947	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2948	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2949	*/
2950	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2951	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2952	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2953	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2954	# undef VMXLOCAL_ADJUST_HOST_SEG
2955	#endif
2956
2957	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2958	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2959	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2960	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2961	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2962	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2963	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2964	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2965	Assert(uSelCS);
2966	Assert(uSelTR);
2967
2968	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2969	#if 0
2970	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2971	Assert(uSelSS != 0);
2972	#endif
2973
2974	/* Write these host selector fields into the host-state area in the VMCS. */
2975	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2976	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2977	#if HC_ARCH_BITS == 64
2978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2979	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2980	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2981	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2982	#else
2983	NOREF(uSelDS);
2984	NOREF(uSelES);
2985	NOREF(uSelFS);
2986	NOREF(uSelGS);
2987	#endif
2988	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2989	AssertRCReturn(rc, rc);
2990
2991	/*
2992	* Host GDTR and IDTR.
2993	*/
2994	RTGDTR Gdtr;
2995	RTIDTR Idtr;
2996	RT_ZERO(Gdtr);
2997	RT_ZERO(Idtr);
2998	ASMGetGDTR(&Gdtr);
2999	ASMGetIDTR(&Idtr);
3000	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3001	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3002	AssertRCReturn(rc, rc);
3003
3004	#if HC_ARCH_BITS == 64
3005	/*
3006	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3007	* maximum limit (0xffff) on every VM-exit.
3008	*/
3009	if (Gdtr.cbGdt != 0xffff)
3010	{
3011	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3012	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3013	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3014	}
3015
3016	/*
3017	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3018	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3019	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3020	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3021	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3022	* hosts where we are pretty sure it won't cause trouble.
3023	*/
3024	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3025	if (Idtr.cbIdt < 0x0fff)
3026	# else
3027	if (Idtr.cbIdt != 0xffff)
3028	# endif
3029	{
3030	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3031	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3032	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3033	}
3034	#endif
3035
3036	/*
3037	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3038	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3039	*/
3040	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3041	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3042	VERR_VMX_INVALID_HOST_STATE);
3043
3044	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3045	#if HC_ARCH_BITS == 64
3046	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3047
3048	/*
3049	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3050	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3051	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3052	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3053	*
3054	* [1] See Intel spec. 3.5 "System Descriptor Types".
3055	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3056	*/
3057	Assert(pDesc->System.u4Type == 11);
3058	if ( pDesc->System.u16LimitLow != 0x67
3059	\|\| pDesc->System.u4LimitHigh)
3060	{
3061	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3062	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3063	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3064	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3065	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3066
3067	/* Store the GDTR here as we need it while restoring TR. */
3068	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3069	}
3070	#else
3071	NOREF(pVM);
3072	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3073	#endif
3074	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3075	AssertRCReturn(rc, rc);
3076
3077	/*
3078	* Host FS base and GS base.
3079	*/
3080	#if HC_ARCH_BITS == 64
3081	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3082	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3083	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3084	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3085	AssertRCReturn(rc, rc);
3086
3087	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3088	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3089	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3090	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3091	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3092	#endif
3093	return rc;
3094	}
3095
3096
3097	/**
3098	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3099	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3100	* the host after every successful VM-exit.
3101	*
3102	* @returns VBox status code.
3103	* @param pVM The cross context VM structure.
3104	* @param pVCpu The cross context virtual CPU structure.
3105	*
3106	* @remarks No-long-jump zone!!!
3107	*/
3108	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3109	{
3110	NOREF(pVM);
3111
3112	AssertPtr(pVCpu);
3113	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3114
3115	/*
3116	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3117	* rather than swapping them on every VM-entry.
3118	*/
3119	hmR0VmxLazySaveHostMsrs(pVCpu);
3120
3121	/*
3122	* Host Sysenter MSRs.
3123	*/
3124	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3125	#if HC_ARCH_BITS == 32
3126	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3127	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3128	#else
3129	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3130	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3131	#endif
3132	AssertRCReturn(rc, rc);
3133
3134	/*
3135	* Host EFER MSR.
3136	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3137	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3138	*/
3139	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3140	{
3141	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3142	AssertRCReturn(rc, rc);
3143	}
3144
3145	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3146	* hmR0VmxLoadGuestExitCtls() !! */
3147
3148	return rc;
3149	}
3150
3151
3152	/**
3153	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3154	*
3155	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3156	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3157	* hmR0VMxLoadGuestEntryCtls().
3158	*
3159	* @returns true if we need to load guest EFER, false otherwise.
3160	* @param pVCpu The cross context virtual CPU structure.
3161	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3162	* out-of-sync. Make sure to update the required fields
3163	* before using them.
3164	*
3165	* @remarks Requires EFER, CR4.
3166	* @remarks No-long-jump zone!!!
3167	*/
3168	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3169	{
3170	#ifdef HMVMX_ALWAYS_SWAP_EFER
3171	return true;
3172	#endif
3173
3174	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3175	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3176	if (CPUMIsGuestInLongMode(pVCpu))
3177	return false;
3178	#endif
3179
3180	PVM pVM = pVCpu->CTX_SUFF(pVM);
3181	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3182	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3183
3184	/*
3185	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3186	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3187	*/
3188	if ( CPUMIsGuestInLongMode(pVCpu)
3189	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3190	{
3191	return true;
3192	}
3193
3194	/*
3195	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3196	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3197	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3198	*/
3199	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3200	&& (pMixedCtx->cr0 & X86_CR0_PG)
3201	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3202	{
3203	/* Assert that host is PAE capable. */
3204	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3205	return true;
3206	}
3207
3208	/** @todo Check the latest Intel spec. for any other bits,
3209	* like SMEP/SMAP? */
3210	return false;
3211	}
3212
3213
3214	/**
3215	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3216	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3217	* controls".
3218	*
3219	* @returns VBox status code.
3220	* @param pVCpu The cross context virtual CPU structure.
3221	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3222	* out-of-sync. Make sure to update the required fields
3223	* before using them.
3224	*
3225	* @remarks Requires EFER.
3226	* @remarks No-long-jump zone!!!
3227	*/
3228	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3229	{
3230	int rc = VINF_SUCCESS;
3231	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3232	{
3233	PVM pVM = pVCpu->CTX_SUFF(pVM);
3234	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3235	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3236
3237	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3238	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3239
3240	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3241	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3242	{
3243	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3244	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3245	}
3246	else
3247	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3248
3249	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3250	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3251	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3252	{
3253	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3254	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3255	}
3256
3257	/*
3258	* The following should -not- be set (since we're not in SMM mode):
3259	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3260	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3261	*/
3262
3263	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3264	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3265
3266	if ((val & zap) != val)
3267	{
3268	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3269	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3270	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3271	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3272	}
3273
3274	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3275	AssertRCReturn(rc, rc);
3276
3277	pVCpu->hm.s.vmx.u32EntryCtls = val;
3278	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3279	}
3280	return rc;
3281	}
3282
3283
3284	/**
3285	* Sets up the VM-exit controls in the VMCS.
3286	*
3287	* @returns VBox status code.
3288	* @param pVCpu The cross context virtual CPU structure.
3289	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3290	* out-of-sync. Make sure to update the required fields
3291	* before using them.
3292	*
3293	* @remarks Requires EFER.
3294	*/
3295	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3296	{
3297	NOREF(pMixedCtx);
3298
3299	int rc = VINF_SUCCESS;
3300	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3301	{
3302	PVM pVM = pVCpu->CTX_SUFF(pVM);
3303	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3304	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3305
3306	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3307	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3308
3309	/*
3310	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3311	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3312	*/
3313	#if HC_ARCH_BITS == 64
3314	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3315	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3316	#else
3317	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3318	{
3319	/* The switcher returns to long mode, EFER is managed by the switcher. */
3320	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3321	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3322	}
3323	else
3324	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3325	#endif
3326
3327	/* If the newer VMCS fields for managing EFER exists, use it. */
3328	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3329	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3330	{
3331	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3332	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3333	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3334	}
3335
3336	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3337	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3338
3339	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3340	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3341	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3342
3343	if ( pVM->hm.s.vmx.fUsePreemptTimer
3344	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3345	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3346
3347	if ((val & zap) != val)
3348	{
3349	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3350	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3351	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3352	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3353	}
3354
3355	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3356	AssertRCReturn(rc, rc);
3357
3358	pVCpu->hm.s.vmx.u32ExitCtls = val;
3359	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3360	}
3361	return rc;
3362	}
3363
3364
3365	/**
3366	* Sets the TPR threshold in the VMCS.
3367	*
3368	* @returns VBox status code.
3369	* @param pVCpu The cross context virtual CPU structure.
3370	* @param u32TprThreshold The TPR threshold (task-priority class only).
3371	*/
3372	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3373	{
3374	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3375	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3376	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3377	}
3378
3379
3380	/**
3381	* Loads the guest APIC and related state.
3382	*
3383	* @returns VBox status code.
3384	* @param pVCpu The cross context virtual CPU structure.
3385	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3386	* out-of-sync. Make sure to update the required fields
3387	* before using them.
3388	*/
3389	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3390	{
3391	NOREF(pMixedCtx);
3392
3393	int rc = VINF_SUCCESS;
3394	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3395	{
3396	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3397	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3398	{
3399	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3400
3401	bool fPendingIntr = false;
3402	uint8_t u8Tpr = 0;
3403	uint8_t u8PendingIntr = 0;
3404	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3405	AssertRCReturn(rc, rc);
3406
3407	/*
3408	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3409	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3410	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3411	* the interrupt when we VM-exit for other reasons.
3412	*/
3413	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3414	uint32_t u32TprThreshold = 0;
3415	if (fPendingIntr)
3416	{
3417	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3418	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3419	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3420	if (u8PendingPriority <= u8TprPriority)
3421	u32TprThreshold = u8PendingPriority;
3422	else
3423	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3424	}
3425
3426	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3427	AssertRCReturn(rc, rc);
3428	}
3429
3430	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3431	}
3432	return rc;
3433	}
3434
3435
3436	/**
3437	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3438	*
3439	* @returns Guest's interruptibility-state.
3440	* @param pVCpu The cross context virtual CPU structure.
3441	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3442	* out-of-sync. Make sure to update the required fields
3443	* before using them.
3444	*
3445	* @remarks No-long-jump zone!!!
3446	*/
3447	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3448	{
3449	/*
3450	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3451	*/
3452	uint32_t uIntrState = 0;
3453	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3454	{
3455	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3456	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3457	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3458	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3459	{
3460	if (pMixedCtx->eflags.Bits.u1IF)
3461	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3462	else
3463	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3464	}
3465	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3466	{
3467	/*
3468	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3469	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3470	*/
3471	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3472	}
3473	}
3474
3475	/*
3476	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3477	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3478	* setting this would block host-NMIs and IRET will not clear the blocking.
3479	*
3480	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3481	*/
3482	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3483	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3484	{
3485	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3486	}
3487
3488	return uIntrState;
3489	}
3490
3491
3492	/**
3493	* Loads the guest's interruptibility-state into the guest-state area in the
3494	* VMCS.
3495	*
3496	* @returns VBox status code.
3497	* @param pVCpu The cross context virtual CPU structure.
3498	* @param uIntrState The interruptibility-state to set.
3499	*/
3500	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3501	{
3502	NOREF(pVCpu);
3503	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3504	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3505	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3506	AssertRC(rc);
3507	return rc;
3508	}
3509
3510
3511	/**
3512	* Loads the exception intercepts required for guest execution in the VMCS.
3513	*
3514	* @returns VBox status code.
3515	* @param pVCpu The cross context virtual CPU structure.
3516	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3517	* out-of-sync. Make sure to update the required fields
3518	* before using them.
3519	*/
3520	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3521	{
3522	NOREF(pMixedCtx);
3523	int rc = VINF_SUCCESS;
3524	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3525	{
3526	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3527	if (pVCpu->hm.s.fGIMTrapXcptUD)
3528	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3529	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530	else
3531	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3532	#endif
3533
3534	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3535	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3536
3537	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3538	AssertRCReturn(rc, rc);
3539
3540	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3541	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3542	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3543	}
3544	return rc;
3545	}
3546
3547
3548	/**
3549	* Loads the guest's RIP into the guest-state area in the VMCS.
3550	*
3551	* @returns VBox status code.
3552	* @param pVCpu The cross context virtual CPU structure.
3553	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3554	* out-of-sync. Make sure to update the required fields
3555	* before using them.
3556	*
3557	* @remarks No-long-jump zone!!!
3558	*/
3559	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3560	{
3561	int rc = VINF_SUCCESS;
3562	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3563	{
3564	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3565	AssertRCReturn(rc, rc);
3566
3567	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3568	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3569	HMCPU_CF_VALUE(pVCpu)));
3570	}
3571	return rc;
3572	}
3573
3574
3575	/**
3576	* Loads the guest's RSP into the guest-state area in the VMCS.
3577	*
3578	* @returns VBox status code.
3579	* @param pVCpu The cross context virtual CPU structure.
3580	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3581	* out-of-sync. Make sure to update the required fields
3582	* before using them.
3583	*
3584	* @remarks No-long-jump zone!!!
3585	*/
3586	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3587	{
3588	int rc = VINF_SUCCESS;
3589	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3590	{
3591	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3592	AssertRCReturn(rc, rc);
3593
3594	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3595	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3596	}
3597	return rc;
3598	}
3599
3600
3601	/**
3602	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3603	*
3604	* @returns VBox status code.
3605	* @param pVCpu The cross context virtual CPU structure.
3606	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3607	* out-of-sync. Make sure to update the required fields
3608	* before using them.
3609	*
3610	* @remarks No-long-jump zone!!!
3611	*/
3612	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3613	{
3614	int rc = VINF_SUCCESS;
3615	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3616	{
3617	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3618	Let us assert it as such and use 32-bit VMWRITE. */
3619	Assert(!(pMixedCtx->rflags.u64 >> 32));
3620	X86EFLAGS Eflags = pMixedCtx->eflags;
3621	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3622	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3623	* These will never be cleared/set, unless some other part of the VMM
3624	* code is buggy - in which case we're better of finding and fixing
3625	* those bugs than hiding them. */
3626	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3627	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3628	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3629	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3630
3631	/*
3632	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3633	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3634	*/
3635	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3636	{
3637	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3638	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3639	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3640	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3641	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3642	}
3643
3644	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3645	AssertRCReturn(rc, rc);
3646
3647	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3648	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3649	}
3650	return rc;
3651	}
3652
3653
3654	/**
3655	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3656	*
3657	* @returns VBox status code.
3658	* @param pVCpu The cross context virtual CPU structure.
3659	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3660	* out-of-sync. Make sure to update the required fields
3661	* before using them.
3662	*
3663	* @remarks No-long-jump zone!!!
3664	*/
3665	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3666	{
3667	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3668	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3669	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3670	AssertRCReturn(rc, rc);
3671	return rc;
3672	}
3673
3674
3675	/**
3676	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3677	* CR0 is partially shared with the host and we have to consider the FPU bits.
3678	*
3679	* @returns VBox status code.
3680	* @param pVCpu The cross context virtual CPU structure.
3681	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3682	* out-of-sync. Make sure to update the required fields
3683	* before using them.
3684	*
3685	* @remarks No-long-jump zone!!!
3686	*/
3687	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3688	{
3689	/*
3690	* Guest CR0.
3691	* Guest FPU.
3692	*/
3693	int rc = VINF_SUCCESS;
3694	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3695	{
3696	Assert(!(pMixedCtx->cr0 >> 32));
3697	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3698	PVM pVM = pVCpu->CTX_SUFF(pVM);
3699
3700	/* The guest's view (read access) of its CR0 is unblemished. */
3701	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3702	AssertRCReturn(rc, rc);
3703	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3704
3705	/* Setup VT-x's view of the guest CR0. */
3706	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3707	if (pVM->hm.s.fNestedPaging)
3708	{
3709	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3710	{
3711	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3712	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3713	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3714	}
3715	else
3716	{
3717	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3718	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3719	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3720	}
3721
3722	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3723	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3724	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3725
3726	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3727	AssertRCReturn(rc, rc);
3728	}
3729	else
3730	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3731
3732	/*
3733	* Guest FPU bits.
3734	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3735	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3736	*/
3737	u32GuestCR0 \|= X86_CR0_NE;
3738	bool fInterceptNM = false;
3739	if (CPUMIsGuestFPUStateActive(pVCpu))
3740	{
3741	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3742	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3743	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3744	}
3745	else
3746	{
3747	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3748	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3749	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3750	}
3751
3752	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3753	bool fInterceptMF = false;
3754	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3755	fInterceptMF = true;
3756
3757	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3758	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3759	{
3760	Assert(PDMVmmDevHeapIsEnabled(pVM));
3761	Assert(pVM->hm.s.vmx.pRealModeTSS);
3762	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3763	fInterceptNM = true;
3764	fInterceptMF = true;
3765	}
3766	else
3767	{
3768	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3769	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3770	}
3771	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3772
3773	if (fInterceptNM)
3774	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3775	else
3776	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3777
3778	if (fInterceptMF)
3779	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3780	else
3781	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3782
3783	/* Additional intercepts for debugging, define these yourself explicitly. */
3784	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3785	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3786	\| RT_BIT(X86_XCPT_BP)
3787	\| RT_BIT(X86_XCPT_DE)
3788	\| RT_BIT(X86_XCPT_NM)
3789	\| RT_BIT(X86_XCPT_TS)
3790	\| RT_BIT(X86_XCPT_UD)
3791	\| RT_BIT(X86_XCPT_NP)
3792	\| RT_BIT(X86_XCPT_SS)
3793	\| RT_BIT(X86_XCPT_GP)
3794	\| RT_BIT(X86_XCPT_PF)
3795	\| RT_BIT(X86_XCPT_MF)
3796	;
3797	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3798	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3799	#endif
3800
3801	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3802
3803	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3804	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3805	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3806	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3807	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3808	else
3809	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3810
3811	u32GuestCR0 \|= uSetCR0;
3812	u32GuestCR0 &= uZapCR0;
3813	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3814
3815	/* Write VT-x's view of the guest CR0 into the VMCS. */
3816	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3817	AssertRCReturn(rc, rc);
3818	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3819	uZapCR0));
3820
3821	/*
3822	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3823	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3824	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3825	*/
3826	uint32_t u32CR0Mask = 0;
3827	u32CR0Mask = X86_CR0_PE
3828	\| X86_CR0_NE
3829	\| X86_CR0_WP
3830	\| X86_CR0_PG
3831	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3832	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3833	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3834
3835	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3836	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3837	* and @bugref{6944}. */
3838	#if 0
3839	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3840	u32CR0Mask &= ~X86_CR0_PE;
3841	#endif
3842	if (pVM->hm.s.fNestedPaging)
3843	u32CR0Mask &= ~X86_CR0_WP;
3844
3845	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3846	if (fInterceptNM)
3847	{
3848	u32CR0Mask \|= X86_CR0_TS
3849	\| X86_CR0_MP;
3850	}
3851
3852	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3853	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3854	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3855	AssertRCReturn(rc, rc);
3856	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3857
3858	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3859	}
3860	return rc;
3861	}
3862
3863
3864	/**
3865	* Loads the guest control registers (CR3, CR4) into the guest-state area
3866	* in the VMCS.
3867	*
3868	* @returns VBox strict status code.
3869	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3870	* without unrestricted guest access and the VMMDev is not presently
3871	* mapped (e.g. EFI32).
3872	*
3873	* @param pVCpu The cross context virtual CPU structure.
3874	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3875	* out-of-sync. Make sure to update the required fields
3876	* before using them.
3877	*
3878	* @remarks No-long-jump zone!!!
3879	*/
3880	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3881	{
3882	int rc = VINF_SUCCESS;
3883	PVM pVM = pVCpu->CTX_SUFF(pVM);
3884
3885	/*
3886	* Guest CR2.
3887	* It's always loaded in the assembler code. Nothing to do here.
3888	*/
3889
3890	/*
3891	* Guest CR3.
3892	*/
3893	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3894	{
3895	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3896	if (pVM->hm.s.fNestedPaging)
3897	{
3898	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3899
3900	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3901	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3902	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3903	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3904
3905	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3906	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3907	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3908
3909	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3910	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3911	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3912	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3913	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3914	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3915	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3916
3917	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3918	AssertRCReturn(rc, rc);
3919	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3920
3921	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3922	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3923	{
3924	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3925	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3926	{
3927	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3928	AssertRCReturn(rc, rc);
3929	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3930	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3931	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3932	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3933	AssertRCReturn(rc, rc);
3934	}
3935
3936	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3937	have Unrestricted Execution to handle the guest when it's not using paging. */
3938	GCPhysGuestCR3 = pMixedCtx->cr3;
3939	}
3940	else
3941	{
3942	/*
3943	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3944	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3945	* EPT takes care of translating it to host-physical addresses.
3946	*/
3947	RTGCPHYS GCPhys;
3948	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3949
3950	/* We obtain it here every time as the guest could have relocated this PCI region. */
3951	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3952	if (RT_SUCCESS(rc))
3953	{ /* likely */ }
3954	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3955	{
3956	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3957	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3958	}
3959	else
3960	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3961
3962	GCPhysGuestCR3 = GCPhys;
3963	}
3964
3965	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3966	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3967	}
3968	else
3969	{
3970	/* Non-nested paging case, just use the hypervisor's CR3. */
3971	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3972
3973	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3974	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3975	}
3976	AssertRCReturn(rc, rc);
3977
3978	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3979	}
3980
3981	/*
3982	* Guest CR4.
3983	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
3984	*/
3985	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3986	{
3987	Assert(!(pMixedCtx->cr4 >> 32));
3988	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3989
3990	/* The guest's view of its CR4 is unblemished. */
3991	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3992	AssertRCReturn(rc, rc);
3993	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3994
3995	/* Setup VT-x's view of the guest CR4. */
3996	/*
3997	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3998	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3999	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4000	*/
4001	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4002	{
4003	Assert(pVM->hm.s.vmx.pRealModeTSS);
4004	Assert(PDMVmmDevHeapIsEnabled(pVM));
4005	u32GuestCR4 &= ~X86_CR4_VME;
4006	}
4007
4008	if (pVM->hm.s.fNestedPaging)
4009	{
4010	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4011	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4012	{
4013	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4014	u32GuestCR4 \|= X86_CR4_PSE;
4015	/* Our identity mapping is a 32-bit page directory. */
4016	u32GuestCR4 &= ~X86_CR4_PAE;
4017	}
4018	/* else use guest CR4.*/
4019	}
4020	else
4021	{
4022	/*
4023	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4024	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4025	*/
4026	switch (pVCpu->hm.s.enmShadowMode)
4027	{
4028	case PGMMODE_REAL: /* Real-mode. */
4029	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4030	case PGMMODE_32_BIT: /* 32-bit paging. */
4031	{
4032	u32GuestCR4 &= ~X86_CR4_PAE;
4033	break;
4034	}
4035
4036	case PGMMODE_PAE: /* PAE paging. */
4037	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4038	{
4039	u32GuestCR4 \|= X86_CR4_PAE;
4040	break;
4041	}
4042
4043	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4044	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4045	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4046	break;
4047	#endif
4048	default:
4049	AssertFailed();
4050	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4051	}
4052	}
4053
4054	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4055	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4056	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4057	u32GuestCR4 \|= uSetCR4;
4058	u32GuestCR4 &= uZapCR4;
4059
4060	/* Write VT-x's view of the guest CR4 into the VMCS. */
4061	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4062	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4063	AssertRCReturn(rc, rc);
4064
4065	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4066	uint32_t u32CR4Mask = X86_CR4_VME
4067	\| X86_CR4_PAE
4068	\| X86_CR4_PGE
4069	\| X86_CR4_PSE
4070	\| X86_CR4_VMXE;
4071	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4072	u32CR4Mask \|= X86_CR4_OSXSAVE;
4073	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4074	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4075	AssertRCReturn(rc, rc);
4076
4077	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4078	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4079
4080	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4081	}
4082	return rc;
4083	}
4084
4085
4086	/**
4087	* Loads the guest debug registers into the guest-state area in the VMCS.
4088	*
4089	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4090	*
4091	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4092	*
4093	* @returns VBox status code.
4094	* @param pVCpu The cross context virtual CPU structure.
4095	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4096	* out-of-sync. Make sure to update the required fields
4097	* before using them.
4098	*
4099	* @remarks No-long-jump zone!!!
4100	*/
4101	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4102	{
4103	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4104	return VINF_SUCCESS;
4105
4106	#ifdef VBOX_STRICT
4107	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4108	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4109	{
4110	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4111	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4112	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4113	}
4114	#endif
4115
4116	int rc;
4117	PVM pVM = pVCpu->CTX_SUFF(pVM);
4118	bool fSteppingDB = false;
4119	bool fInterceptMovDRx = false;
4120	if (pVCpu->hm.s.fSingleInstruction)
4121	{
4122	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4123	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4124	{
4125	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4126	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4127	AssertRCReturn(rc, rc);
4128	Assert(fSteppingDB == false);
4129	}
4130	else
4131	{
4132	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4133	pVCpu->hm.s.fClearTrapFlag = true;
4134	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4135	fSteppingDB = true;
4136	}
4137	}
4138
4139	if ( fSteppingDB
4140	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4141	{
4142	/*
4143	* Use the combined guest and host DRx values found in the hypervisor
4144	* register set because the debugger has breakpoints active or someone
4145	* is single stepping on the host side without a monitor trap flag.
4146	*
4147	* Note! DBGF expects a clean DR6 state before executing guest code.
4148	*/
4149	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4150	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4151	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4152	{
4153	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4154	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4155	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4156	}
4157	else
4158	#endif
4159	if (!CPUMIsHyperDebugStateActive(pVCpu))
4160	{
4161	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4162	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4163	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4164	}
4165
4166	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4167	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4168	AssertRCReturn(rc, rc);
4169
4170	pVCpu->hm.s.fUsingHyperDR7 = true;
4171	fInterceptMovDRx = true;
4172	}
4173	else
4174	{
4175	/*
4176	* If the guest has enabled debug registers, we need to load them prior to
4177	* executing guest code so they'll trigger at the right time.
4178	*/
4179	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4180	{
4181	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4182	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4183	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4184	{
4185	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4186	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4187	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4188	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4189	}
4190	else
4191	#endif
4192	if (!CPUMIsGuestDebugStateActive(pVCpu))
4193	{
4194	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4195	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4196	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4197	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4198	}
4199	Assert(!fInterceptMovDRx);
4200	}
4201	/*
4202	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4203	* must intercept #DB in order to maintain a correct DR6 guest value, and
4204	* because we need to intercept it to prevent nested #DBs from hanging the
4205	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4206	*/
4207	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4208	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4209	&& !CPUMIsGuestDebugStateActive(pVCpu))
4210	#else
4211	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4212	#endif
4213	{
4214	fInterceptMovDRx = true;
4215	}
4216
4217	/* Update guest DR7. */
4218	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4219	AssertRCReturn(rc, rc);
4220
4221	pVCpu->hm.s.fUsingHyperDR7 = false;
4222	}
4223
4224	/*
4225	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4226	*/
4227	if (fInterceptMovDRx)
4228	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4229	else
4230	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4231	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4232	AssertRCReturn(rc, rc);
4233
4234	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4235	return VINF_SUCCESS;
4236	}
4237
4238
4239	#ifdef VBOX_STRICT
4240	/**
4241	* Strict function to validate segment registers.
4242	*
4243	* @remarks ASSUMES CR0 is up to date.
4244	*/
4245	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4246	{
4247	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4248	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4249	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4250	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4251	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4252	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4253	{
4254	/* Protected mode checks */
4255	/* CS */
4256	Assert(pCtx->cs.Attr.n.u1Present);
4257	Assert(!(pCtx->cs.Attr.u & 0xf00));
4258	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4259	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4260	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4261	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4262	\|\| (pCtx->cs.Attr.n.u1Granularity));
4263	/* CS cannot be loaded with NULL in protected mode. */
4264	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4265	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4266	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4267	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4268	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4269	else
4270	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4271	/* SS */
4272	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4273	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4274	if ( !(pCtx->cr0 & X86_CR0_PE)
4275	\|\| pCtx->cs.Attr.n.u4Type == 3)
4276	{
4277	Assert(!pCtx->ss.Attr.n.u2Dpl);
4278	}
4279	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4280	{
4281	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4283	Assert(pCtx->ss.Attr.n.u1Present);
4284	Assert(!(pCtx->ss.Attr.u & 0xf00));
4285	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4286	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4287	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4288	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4289	\|\| (pCtx->ss.Attr.n.u1Granularity));
4290	}
4291	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4292	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4293	{
4294	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4295	Assert(pCtx->ds.Attr.n.u1Present);
4296	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4297	Assert(!(pCtx->ds.Attr.u & 0xf00));
4298	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4299	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4300	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4301	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4302	\|\| (pCtx->ds.Attr.n.u1Granularity));
4303	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4304	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4305	}
4306	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4307	{
4308	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4309	Assert(pCtx->es.Attr.n.u1Present);
4310	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4311	Assert(!(pCtx->es.Attr.u & 0xf00));
4312	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4313	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4314	\|\| !(pCtx->es.Attr.n.u1Granularity));
4315	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4316	\|\| (pCtx->es.Attr.n.u1Granularity));
4317	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4318	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4319	}
4320	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4321	{
4322	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4323	Assert(pCtx->fs.Attr.n.u1Present);
4324	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4325	Assert(!(pCtx->fs.Attr.u & 0xf00));
4326	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4327	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4328	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4329	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4330	\|\| (pCtx->fs.Attr.n.u1Granularity));
4331	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4332	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4333	}
4334	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4335	{
4336	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4337	Assert(pCtx->gs.Attr.n.u1Present);
4338	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4339	Assert(!(pCtx->gs.Attr.u & 0xf00));
4340	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4341	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4342	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4343	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4344	\|\| (pCtx->gs.Attr.n.u1Granularity));
4345	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4346	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4347	}
4348	/* 64-bit capable CPUs. */
4349	# if HC_ARCH_BITS == 64
4350	Assert(!(pCtx->cs.u64Base >> 32));
4351	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4352	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4353	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4354	# endif
4355	}
4356	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4357	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4358	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4359	{
4360	/* Real and v86 mode checks. */
4361	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4362	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4363	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4364	{
4365	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4366	}
4367	else
4368	{
4369	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4370	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4371	}
4372
4373	/* CS */
4374	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4375	Assert(pCtx->cs.u32Limit == 0xffff);
4376	Assert(u32CSAttr == 0xf3);
4377	/* SS */
4378	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4379	Assert(pCtx->ss.u32Limit == 0xffff);
4380	Assert(u32SSAttr == 0xf3);
4381	/* DS */
4382	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4383	Assert(pCtx->ds.u32Limit == 0xffff);
4384	Assert(u32DSAttr == 0xf3);
4385	/* ES */
4386	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4387	Assert(pCtx->es.u32Limit == 0xffff);
4388	Assert(u32ESAttr == 0xf3);
4389	/* FS */
4390	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4391	Assert(pCtx->fs.u32Limit == 0xffff);
4392	Assert(u32FSAttr == 0xf3);
4393	/* GS */
4394	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4395	Assert(pCtx->gs.u32Limit == 0xffff);
4396	Assert(u32GSAttr == 0xf3);
4397	/* 64-bit capable CPUs. */
4398	# if HC_ARCH_BITS == 64
4399	Assert(!(pCtx->cs.u64Base >> 32));
4400	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4401	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4402	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4403	# endif
4404	}
4405	}
4406	#endif /* VBOX_STRICT */
4407
4408
4409	/**
4410	* Writes a guest segment register into the guest-state area in the VMCS.
4411	*
4412	* @returns VBox status code.
4413	* @param pVCpu The cross context virtual CPU structure.
4414	* @param idxSel Index of the selector in the VMCS.
4415	* @param idxLimit Index of the segment limit in the VMCS.
4416	* @param idxBase Index of the segment base in the VMCS.
4417	* @param idxAccess Index of the access rights of the segment in the VMCS.
4418	* @param pSelReg Pointer to the segment selector.
4419	*
4420	* @remarks No-long-jump zone!!!
4421	*/
4422	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4423	uint32_t idxAccess, PCPUMSELREG pSelReg)
4424	{
4425	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4426	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4427	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4428	AssertRCReturn(rc, rc);
4429
4430	uint32_t u32Access = pSelReg->Attr.u;
4431	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4432	{
4433	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4434	u32Access = 0xf3;
4435	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4436	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4437	}
4438	else
4439	{
4440	/*
4441	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4442	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4443	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4444	* loaded in protected-mode have their attribute as 0.
4445	*/
4446	if (!u32Access)
4447	u32Access = X86DESCATTR_UNUSABLE;
4448	}
4449
4450	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4451	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4452	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4453
4454	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4455	AssertRCReturn(rc, rc);
4456	return rc;
4457	}
4458
4459
4460	/**
4461	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4462	* into the guest-state area in the VMCS.
4463	*
4464	* @returns VBox status code.
4465	* @param pVCpu The cross context virtual CPU structure.
4466	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4467	* out-of-sync. Make sure to update the required fields
4468	* before using them.
4469	*
4470	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4471	* @remarks No-long-jump zone!!!
4472	*/
4473	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4474	{
4475	int rc = VERR_INTERNAL_ERROR_5;
4476	PVM pVM = pVCpu->CTX_SUFF(pVM);
4477
4478	/*
4479	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4480	*/
4481	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4482	{
4483	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4484	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4485	{
4486	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4487	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4488	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4489	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4490	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4491	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4492	}
4493
4494	#ifdef VBOX_WITH_REM
4495	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4496	{
4497	Assert(pVM->hm.s.vmx.pRealModeTSS);
4498	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4499	if ( pVCpu->hm.s.vmx.fWasInRealMode
4500	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4501	{
4502	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4503	in real-mode (e.g. OpenBSD 4.0) */
4504	REMFlushTBs(pVM);
4505	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4506	pVCpu->hm.s.vmx.fWasInRealMode = false;
4507	}
4508	}
4509	#endif
4510	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4511	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4512	AssertRCReturn(rc, rc);
4513	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4514	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4515	AssertRCReturn(rc, rc);
4516	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4517	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4518	AssertRCReturn(rc, rc);
4519	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4520	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4521	AssertRCReturn(rc, rc);
4522	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4523	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4524	AssertRCReturn(rc, rc);
4525	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4526	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4527	AssertRCReturn(rc, rc);
4528
4529	#ifdef VBOX_STRICT
4530	/* Validate. */
4531	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4532	#endif
4533
4534	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4535	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4536	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4537	}
4538
4539	/*
4540	* Guest TR.
4541	*/
4542	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4543	{
4544	/*
4545	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4546	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4547	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4548	*/
4549	uint16_t u16Sel = 0;
4550	uint32_t u32Limit = 0;
4551	uint64_t u64Base = 0;
4552	uint32_t u32AccessRights = 0;
4553
4554	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4555	{
4556	u16Sel = pMixedCtx->tr.Sel;
4557	u32Limit = pMixedCtx->tr.u32Limit;
4558	u64Base = pMixedCtx->tr.u64Base;
4559	u32AccessRights = pMixedCtx->tr.Attr.u;
4560	}
4561	else
4562	{
4563	Assert(pVM->hm.s.vmx.pRealModeTSS);
4564	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4565
4566	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4567	RTGCPHYS GCPhys;
4568	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4569	AssertRCReturn(rc, rc);
4570
4571	X86DESCATTR DescAttr;
4572	DescAttr.u = 0;
4573	DescAttr.n.u1Present = 1;
4574	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4575
4576	u16Sel = 0;
4577	u32Limit = HM_VTX_TSS_SIZE;
4578	u64Base = GCPhys; /* in real-mode phys = virt. */
4579	u32AccessRights = DescAttr.u;
4580	}
4581
4582	/* Validate. */
4583	Assert(!(u16Sel & RT_BIT(2)));
4584	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4585	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4586	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4587	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4588	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4589	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4590	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4591	Assert( (u32Limit & 0xfff) == 0xfff
4592	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4593	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4594	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4595
4596	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4597	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4598	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4599	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4600	AssertRCReturn(rc, rc);
4601
4602	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4603	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4604	}
4605
4606	/*
4607	* Guest GDTR.
4608	*/
4609	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4610	{
4611	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4612	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4613	AssertRCReturn(rc, rc);
4614
4615	/* Validate. */
4616	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4617
4618	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4619	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4620	}
4621
4622	/*
4623	* Guest LDTR.
4624	*/
4625	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4626	{
4627	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4628	uint32_t u32Access = 0;
4629	if (!pMixedCtx->ldtr.Attr.u)
4630	u32Access = X86DESCATTR_UNUSABLE;
4631	else
4632	u32Access = pMixedCtx->ldtr.Attr.u;
4633
4634	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4635	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4636	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4637	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4638	AssertRCReturn(rc, rc);
4639
4640	/* Validate. */
4641	if (!(u32Access & X86DESCATTR_UNUSABLE))
4642	{
4643	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4644	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4645	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4646	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4647	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4648	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4649	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4650	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4651	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4652	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4653	}
4654
4655	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4656	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4657	}
4658
4659	/*
4660	* Guest IDTR.
4661	*/
4662	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4663	{
4664	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4665	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4666	AssertRCReturn(rc, rc);
4667
4668	/* Validate. */
4669	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4670
4671	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4672	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4673	}
4674
4675	return VINF_SUCCESS;
4676	}
4677
4678
4679	/**
4680	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4681	* areas.
4682	*
4683	* These MSRs will automatically be loaded to the host CPU on every successful
4684	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4685	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4686	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4687	*
4688	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4689	*
4690	* @returns VBox status code.
4691	* @param pVCpu The cross context virtual CPU structure.
4692	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4693	* out-of-sync. Make sure to update the required fields
4694	* before using them.
4695	*
4696	* @remarks No-long-jump zone!!!
4697	*/
4698	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4699	{
4700	AssertPtr(pVCpu);
4701	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4702
4703	/*
4704	* MSRs that we use the auto-load/store MSR area in the VMCS.
4705	*/
4706	PVM pVM = pVCpu->CTX_SUFF(pVM);
4707	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4708	{
4709	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4710	#if HC_ARCH_BITS == 32
4711	if (pVM->hm.s.fAllow64BitGuests)
4712	{
4713	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4714	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4715	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4716	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4717	AssertRCReturn(rc, rc);
4718	# ifdef LOG_ENABLED
4719	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4720	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4721	{
4722	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4723	pMsr->u64Value));
4724	}
4725	# endif
4726	}
4727	#endif
4728	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4729	}
4730
4731	/*
4732	* Guest Sysenter MSRs.
4733	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4734	* VM-exits on WRMSRs for these MSRs.
4735	*/
4736	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4737	{
4738	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4739	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4740	}
4741
4742	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4743	{
4744	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4745	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4746	}
4747
4748	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4749	{
4750	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4751	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4752	}
4753
4754	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4755	{
4756	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4757	{
4758	/*
4759	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4760	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4761	*/
4762	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4763	{
4764	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4765	AssertRCReturn(rc,rc);
4766	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4767	}
4768	else
4769	{
4770	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4771	NULL /* pfAddedAndUpdated */);
4772	AssertRCReturn(rc, rc);
4773
4774	/* We need to intercept reads too, see @bugref{7386#c16}. */
4775	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4776	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4777	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4778	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4779	}
4780	}
4781	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4782	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4783	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4784	}
4785
4786	return VINF_SUCCESS;
4787	}
4788
4789
4790	/**
4791	* Loads the guest activity state into the guest-state area in the VMCS.
4792	*
4793	* @returns VBox status code.
4794	* @param pVCpu The cross context virtual CPU structure.
4795	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4796	* out-of-sync. Make sure to update the required fields
4797	* before using them.
4798	*
4799	* @remarks No-long-jump zone!!!
4800	*/
4801	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4802	{
4803	NOREF(pMixedCtx);
4804	/** @todo See if we can make use of other states, e.g.
4805	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4806	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4807	{
4808	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4809	AssertRCReturn(rc, rc);
4810
4811	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4812	}
4813	return VINF_SUCCESS;
4814	}
4815
4816
4817	/**
4818	* Sets up the appropriate function to run guest code.
4819	*
4820	* @returns VBox status code.
4821	* @param pVCpu The cross context virtual CPU structure.
4822	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4823	* out-of-sync. Make sure to update the required fields
4824	* before using them.
4825	*
4826	* @remarks No-long-jump zone!!!
4827	*/
4828	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4829	{
4830	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4831	{
4832	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4833	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4834	#endif
4835	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4836	#if HC_ARCH_BITS == 32
4837	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4838	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4839	{
4840	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4841	{
4842	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4843	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4844	\| HM_CHANGED_VMX_ENTRY_CTLS
4845	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4846	}
4847	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4848	}
4849	#else
4850	/* 64-bit host. */
4851	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4852	#endif
4853	}
4854	else
4855	{
4856	/* Guest is not in long mode, use the 32-bit handler. */
4857	#if HC_ARCH_BITS == 32
4858	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4859	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4860	{
4861	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4862	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4863	\| HM_CHANGED_VMX_ENTRY_CTLS
4864	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4865	}
4866	#endif
4867	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4868	}
4869	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4870	return VINF_SUCCESS;
4871	}
4872
4873
4874	/**
4875	* Wrapper for running the guest code in VT-x.
4876	*
4877	* @returns VBox status code, no informational status codes.
4878	* @param pVM The cross context VM structure.
4879	* @param pVCpu The cross context virtual CPU structure.
4880	* @param pCtx Pointer to the guest-CPU context.
4881	*
4882	* @remarks No-long-jump zone!!!
4883	*/
4884	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4885	{
4886	/*
4887	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4888	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4889	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4890	*/
4891	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4892	/** @todo Add stats for resume vs launch. */
4893	#ifdef VBOX_WITH_KERNEL_USING_XMM
4894	int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4895	#else
4896	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4897	#endif
4898	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4899	return rc;
4900	}
4901
4902
4903	/**
4904	* Reports world-switch error and dumps some useful debug info.
4905	*
4906	* @param pVM The cross context VM structure.
4907	* @param pVCpu The cross context virtual CPU structure.
4908	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4909	* @param pCtx Pointer to the guest-CPU context.
4910	* @param pVmxTransient Pointer to the VMX transient structure (only
4911	* exitReason updated).
4912	*/
4913	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4914	{
4915	Assert(pVM);
4916	Assert(pVCpu);
4917	Assert(pCtx);
4918	Assert(pVmxTransient);
4919	HMVMX_ASSERT_PREEMPT_SAFE();
4920
4921	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4922	switch (rcVMRun)
4923	{
4924	case VERR_VMX_INVALID_VMXON_PTR:
4925	AssertFailed();
4926	break;
4927	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4928	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4929	{
4930	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4931	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4932	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4933	AssertRC(rc);
4934
4935	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4936	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4937	Cannot do it here as we may have been long preempted. */
4938
4939	#ifdef VBOX_STRICT
4940	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4941	pVmxTransient->uExitReason));
4942	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4943	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4944	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4945	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4946	else
4947	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4948	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4949	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4950
4951	/* VMX control bits. */
4952	uint32_t u32Val;
4953	uint64_t u64Val;
4954	RTHCUINTREG uHCReg;
4955	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4956	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4957	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4958	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4959	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4960	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4961	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4962	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4963	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4964	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4965	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4966	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4967	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4968	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4969	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4970	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4971	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4972	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4973	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4974	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4975	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4976	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4977	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4978	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4979	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4980	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4981	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4982	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4983	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4984	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4985	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4986	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4987	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4988	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4989	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4990	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4991	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4992	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4993	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4994	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4995	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4996	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4997
4998	/* Guest bits. */
4999	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5000	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5001	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5002	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5003	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5004	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5005	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5006	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5007
5008	/* Host bits. */
5009	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5010	Log4(("Host CR0 %#RHr\n", uHCReg));
5011	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5012	Log4(("Host CR3 %#RHr\n", uHCReg));
5013	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5014	Log4(("Host CR4 %#RHr\n", uHCReg));
5015
5016	RTGDTR HostGdtr;
5017	PCX86DESCHC pDesc;
5018	ASMGetGDTR(&HostGdtr);
5019	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5020	Log4(("Host CS %#08x\n", u32Val));
5021	if (u32Val < HostGdtr.cbGdt)
5022	{
5023	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5024	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5025	}
5026
5027	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5028	Log4(("Host DS %#08x\n", u32Val));
5029	if (u32Val < HostGdtr.cbGdt)
5030	{
5031	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5032	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5033	}
5034
5035	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5036	Log4(("Host ES %#08x\n", u32Val));
5037	if (u32Val < HostGdtr.cbGdt)
5038	{
5039	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5040	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5041	}
5042
5043	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5044	Log4(("Host FS %#08x\n", u32Val));
5045	if (u32Val < HostGdtr.cbGdt)
5046	{
5047	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5048	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5049	}
5050
5051	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5052	Log4(("Host GS %#08x\n", u32Val));
5053	if (u32Val < HostGdtr.cbGdt)
5054	{
5055	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5056	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5057	}
5058
5059	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5060	Log4(("Host SS %#08x\n", u32Val));
5061	if (u32Val < HostGdtr.cbGdt)
5062	{
5063	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5064	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5065	}
5066
5067	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5068	Log4(("Host TR %#08x\n", u32Val));
5069	if (u32Val < HostGdtr.cbGdt)
5070	{
5071	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5072	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5073	}
5074
5075	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5076	Log4(("Host TR Base %#RHv\n", uHCReg));
5077	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5078	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5079	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5080	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5081	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5082	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5083	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5084	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5085	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5086	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5087	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5088	Log4(("Host RSP %#RHv\n", uHCReg));
5089	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5090	Log4(("Host RIP %#RHv\n", uHCReg));
5091	# if HC_ARCH_BITS == 64
5092	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5093	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5094	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5095	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5096	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5097	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5098	# endif
5099	#endif /* VBOX_STRICT */
5100	break;
5101	}
5102
5103	default:
5104	/* Impossible */
5105	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5106	break;
5107	}
5108	NOREF(pVM); NOREF(pCtx);
5109	}
5110
5111
5112	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5113	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5114	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5115	#endif
5116	#ifdef VBOX_STRICT
5117	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5118	{
5119	switch (idxField)
5120	{
5121	case VMX_VMCS_GUEST_RIP:
5122	case VMX_VMCS_GUEST_RSP:
5123	case VMX_VMCS_GUEST_SYSENTER_EIP:
5124	case VMX_VMCS_GUEST_SYSENTER_ESP:
5125	case VMX_VMCS_GUEST_GDTR_BASE:
5126	case VMX_VMCS_GUEST_IDTR_BASE:
5127	case VMX_VMCS_GUEST_CS_BASE:
5128	case VMX_VMCS_GUEST_DS_BASE:
5129	case VMX_VMCS_GUEST_ES_BASE:
5130	case VMX_VMCS_GUEST_FS_BASE:
5131	case VMX_VMCS_GUEST_GS_BASE:
5132	case VMX_VMCS_GUEST_SS_BASE:
5133	case VMX_VMCS_GUEST_LDTR_BASE:
5134	case VMX_VMCS_GUEST_TR_BASE:
5135	case VMX_VMCS_GUEST_CR3:
5136	return true;
5137	}
5138	return false;
5139	}
5140
5141	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5142	{
5143	switch (idxField)
5144	{
5145	/* Read-only fields. */
5146	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5147	return true;
5148	}
5149	/* Remaining readable fields should also be writable. */
5150	return hmR0VmxIsValidWriteField(idxField);
5151	}
5152	#endif /* VBOX_STRICT */
5153
5154
5155	/**
5156	* Executes the specified handler in 64-bit mode.
5157	*
5158	* @returns VBox status code (no informational status codes).
5159	* @param pVM The cross context VM structure.
5160	* @param pVCpu The cross context virtual CPU structure.
5161	* @param pCtx Pointer to the guest CPU context.
5162	* @param enmOp The operation to perform.
5163	* @param cParams Number of parameters.
5164	* @param paParam Array of 32-bit parameters.
5165	*/
5166	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5167	uint32_t cParams, uint32_t *paParam)
5168	{
5169	NOREF(pCtx);
5170
5171	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5172	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5173	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5174	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5175
5176	#ifdef VBOX_STRICT
5177	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5178	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5179
5180	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5181	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5182	#endif
5183
5184	/* Disable interrupts. */
5185	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5186
5187	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5188	RTCPUID idHostCpu = RTMpCpuId();
5189	CPUMR0SetLApic(pVCpu, idHostCpu);
5190	#endif
5191
5192	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5193	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5194
5195	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5196	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5197
5198	/* Leave VMX Root Mode. */
5199	VMXDisable();
5200
5201	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5202
5203	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5204	CPUMSetHyperEIP(pVCpu, enmOp);
5205	for (int i = (int)cParams - 1; i >= 0; i--)
5206	CPUMPushHyper(pVCpu, paParam[i]);
5207
5208	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5209
5210	/* Call the switcher. */
5211	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5212	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5213
5214	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5215	/* Make sure the VMX instructions don't cause #UD faults. */
5216	SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5217
5218	/* Re-enter VMX Root Mode */
5219	int rc2 = VMXEnable(HCPhysCpuPage);
5220	if (RT_FAILURE(rc2))
5221	{
5222	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5223	ASMSetFlags(fOldEFlags);
5224	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5225	return rc2;
5226	}
5227
5228	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5229	AssertRC(rc2);
5230	Assert(!(ASMGetFlags() & X86_EFL_IF));
5231	ASMSetFlags(fOldEFlags);
5232	return rc;
5233	}
5234
5235
5236	/**
5237	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5238	* supporting 64-bit guests.
5239	*
5240	* @returns VBox status code.
5241	* @param fResume Whether to VMLAUNCH or VMRESUME.
5242	* @param pCtx Pointer to the guest-CPU context.
5243	* @param pCache Pointer to the VMCS cache.
5244	* @param pVM The cross context VM structure.
5245	* @param pVCpu The cross context virtual CPU structure.
5246	*/
5247	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5248	{
5249	NOREF(fResume);
5250
5251	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5252	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5253
5254	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5255	pCache->uPos = 1;
5256	pCache->interPD = PGMGetInterPaeCR3(pVM);
5257	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5258	#endif
5259
5260	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5261	pCache->TestIn.HCPhysCpuPage = 0;
5262	pCache->TestIn.HCPhysVmcs = 0;
5263	pCache->TestIn.pCache = 0;
5264	pCache->TestOut.HCPhysVmcs = 0;
5265	pCache->TestOut.pCache = 0;
5266	pCache->TestOut.pCtx = 0;
5267	pCache->TestOut.eflags = 0;
5268	#else
5269	NOREF(pCache);
5270	#endif
5271
5272	uint32_t aParam[10];
5273	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5274	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5275	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5276	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5277	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5278	aParam[5] = 0;
5279	aParam[6] = VM_RC_ADDR(pVM, pVM);
5280	aParam[7] = 0;
5281	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5282	aParam[9] = 0;
5283
5284	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5285	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5286	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5287	#endif
5288	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5289
5290	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5291	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5292	Assert(pCtx->dr[4] == 10);
5293	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5294	#endif
5295
5296	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5297	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5298	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5299	pVCpu->hm.s.vmx.HCPhysVmcs));
5300	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5301	pCache->TestOut.HCPhysVmcs));
5302	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5303	pCache->TestOut.pCache));
5304	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5305	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5306	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5307	pCache->TestOut.pCtx));
5308	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5309	#endif
5310	return rc;
5311	}
5312
5313
5314	/**
5315	* Initialize the VMCS-Read cache.
5316	*
5317	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5318	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5319	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5320	* (those that have a 32-bit FULL & HIGH part).
5321	*
5322	* @returns VBox status code.
5323	* @param pVM The cross context VM structure.
5324	* @param pVCpu The cross context virtual CPU structure.
5325	*/
5326	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5327	{
5328	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5329	{ \
5330	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5331	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5332	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5333	++cReadFields; \
5334	}
5335
5336	AssertPtr(pVM);
5337	AssertPtr(pVCpu);
5338	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5339	uint32_t cReadFields = 0;
5340
5341	/*
5342	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5343	* and serve to indicate exceptions to the rules.
5344	*/
5345
5346	/* Guest-natural selector base fields. */
5347	#if 0
5348	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5349	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5350	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5351	#endif
5352	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5353	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5354	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5355	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5356	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5357	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5358	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5359	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5360	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5361	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5362	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5363	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5364	#if 0
5365	/* Unused natural width guest-state fields. */
5366	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5367	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5368	#endif
5369	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5370	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5371
5372	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5373	#if 0
5374	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5375	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5376	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5377	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5378	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5379	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5380	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5381	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5382	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5383	#endif
5384
5385	/* Natural width guest-state fields. */
5386	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5387	#if 0
5388	/* Currently unused field. */
5389	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5390	#endif
5391
5392	if (pVM->hm.s.fNestedPaging)
5393	{
5394	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5395	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5396	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5397	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5398	}
5399	else
5400	{
5401	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5402	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5403	}
5404
5405	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5406	return VINF_SUCCESS;
5407	}
5408
5409
5410	/**
5411	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5412	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5413	* darwin, running 64-bit guests).
5414	*
5415	* @returns VBox status code.
5416	* @param pVCpu The cross context virtual CPU structure.
5417	* @param idxField The VMCS field encoding.
5418	* @param u64Val 16, 32 or 64-bit value.
5419	*/
5420	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5421	{
5422	int rc;
5423	switch (idxField)
5424	{
5425	/*
5426	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5427	*/
5428	/* 64-bit Control fields. */
5429	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5430	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5431	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5432	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5433	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5434	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5435	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5436	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5437	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5438	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5439	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5440	case VMX_VMCS64_CTRL_EPTP_FULL:
5441	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5442	/* 64-bit Guest-state fields. */
5443	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5444	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5445	case VMX_VMCS64_GUEST_PAT_FULL:
5446	case VMX_VMCS64_GUEST_EFER_FULL:
5447	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5448	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5449	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5450	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5451	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5452	/* 64-bit Host-state fields. */
5453	case VMX_VMCS64_HOST_PAT_FULL:
5454	case VMX_VMCS64_HOST_EFER_FULL:
5455	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5456	{
5457	rc = VMXWriteVmcs32(idxField, u64Val);
5458	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5459	break;
5460	}
5461
5462	/*
5463	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5464	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5465	*/
5466	/* Natural-width Guest-state fields. */
5467	case VMX_VMCS_GUEST_CR3:
5468	case VMX_VMCS_GUEST_ES_BASE:
5469	case VMX_VMCS_GUEST_CS_BASE:
5470	case VMX_VMCS_GUEST_SS_BASE:
5471	case VMX_VMCS_GUEST_DS_BASE:
5472	case VMX_VMCS_GUEST_FS_BASE:
5473	case VMX_VMCS_GUEST_GS_BASE:
5474	case VMX_VMCS_GUEST_LDTR_BASE:
5475	case VMX_VMCS_GUEST_TR_BASE:
5476	case VMX_VMCS_GUEST_GDTR_BASE:
5477	case VMX_VMCS_GUEST_IDTR_BASE:
5478	case VMX_VMCS_GUEST_RSP:
5479	case VMX_VMCS_GUEST_RIP:
5480	case VMX_VMCS_GUEST_SYSENTER_ESP:
5481	case VMX_VMCS_GUEST_SYSENTER_EIP:
5482	{
5483	if (!(u64Val >> 32))
5484	{
5485	/* If this field is 64-bit, VT-x will zero out the top bits. */
5486	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5487	}
5488	else
5489	{
5490	/* Assert that only the 32->64 switcher case should ever come here. */
5491	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5492	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5493	}
5494	break;
5495	}
5496
5497	default:
5498	{
5499	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5500	rc = VERR_INVALID_PARAMETER;
5501	break;
5502	}
5503	}
5504	AssertRCReturn(rc, rc);
5505	return rc;
5506	}
5507
5508
5509	/**
5510	* Queue up a VMWRITE by using the VMCS write cache.
5511	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5512	*
5513	* @param pVCpu The cross context virtual CPU structure.
5514	* @param idxField The VMCS field encoding.
5515	* @param u64Val 16, 32 or 64-bit value.
5516	*/
5517	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5518	{
5519	AssertPtr(pVCpu);
5520	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5521
5522	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5523	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5524
5525	/* Make sure there are no duplicates. */
5526	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5527	{
5528	if (pCache->Write.aField[i] == idxField)
5529	{
5530	pCache->Write.aFieldVal[i] = u64Val;
5531	return VINF_SUCCESS;
5532	}
5533	}
5534
5535	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5536	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5537	pCache->Write.cValidEntries++;
5538	return VINF_SUCCESS;
5539	}
5540	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5541
5542
5543	/**
5544	* Sets up the usage of TSC-offsetting and updates the VMCS.
5545	*
5546	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5547	* VMX preemption timer.
5548	*
5549	* @returns VBox status code.
5550	* @param pVM The cross context VM structure.
5551	* @param pVCpu The cross context virtual CPU structure.
5552	*
5553	* @remarks No-long-jump zone!!!
5554	*/
5555	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5556	{
5557	int rc;
5558	bool fOffsettedTsc;
5559	bool fParavirtTsc;
5560	if (pVM->hm.s.vmx.fUsePreemptTimer)
5561	{
5562	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5563	&fOffsettedTsc, &fParavirtTsc);
5564
5565	/* Make sure the returned values have sane upper and lower boundaries. */
5566	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5567	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5568	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5569	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5570
5571	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5572	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5573	}
5574	else
5575	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5576
5577	/** @todo later optimize this to be done elsewhere and not before every
5578	* VM-entry. */
5579	if (fParavirtTsc)
5580	{
5581	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5582	information before every VM-entry, hence disable it for performance sake. */
5583	#if 0
5584	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5585	AssertRC(rc);
5586	#endif
5587	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5588	}
5589
5590	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5591	{
5592	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5593	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5594
5595	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5596	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5597	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5598	}
5599	else
5600	{
5601	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5602	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5603	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5604	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5605	}
5606	}
5607
5608
5609	/**
5610	* Determines if an exception is a contributory exception.
5611	*
5612	* Contributory exceptions are ones which can cause double-faults unless the
5613	* original exception was a benign exception. Page-fault is intentionally not
5614	* included here as it's a conditional contributory exception.
5615	*
5616	* @returns true if the exception is contributory, false otherwise.
5617	* @param uVector The exception vector.
5618	*/
5619	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5620	{
5621	switch (uVector)
5622	{
5623	case X86_XCPT_GP:
5624	case X86_XCPT_SS:
5625	case X86_XCPT_NP:
5626	case X86_XCPT_TS:
5627	case X86_XCPT_DE:
5628	return true;
5629	default:
5630	break;
5631	}
5632	return false;
5633	}
5634
5635
5636	/**
5637	* Sets an event as a pending event to be injected into the guest.
5638	*
5639	* @param pVCpu The cross context virtual CPU structure.
5640	* @param u32IntInfo The VM-entry interruption-information field.
5641	* @param cbInstr The VM-entry instruction length in bytes (for software
5642	* interrupts, exceptions and privileged software
5643	* exceptions).
5644	* @param u32ErrCode The VM-entry exception error code.
5645	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5646	* page-fault.
5647	*
5648	* @remarks Statistics counter assumes this is a guest event being injected or
5649	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5650	* always incremented.
5651	*/
5652	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5653	RTGCUINTPTR GCPtrFaultAddress)
5654	{
5655	Assert(!pVCpu->hm.s.Event.fPending);
5656	pVCpu->hm.s.Event.fPending = true;
5657	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5658	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5659	pVCpu->hm.s.Event.cbInstr = cbInstr;
5660	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5661
5662	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5663	}
5664
5665
5666	/**
5667	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5668	*
5669	* @param pVCpu The cross context virtual CPU structure.
5670	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5671	* out-of-sync. Make sure to update the required fields
5672	* before using them.
5673	*/
5674	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5675	{
5676	NOREF(pMixedCtx);
5677	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5678	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5679	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5680	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5681	}
5682
5683
5684	/**
5685	* Handle a condition that occurred while delivering an event through the guest
5686	* IDT.
5687	*
5688	* @returns Strict VBox status code (i.e. informational status codes too).
5689	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5690	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5691	* to continue execution of the guest which will delivery the \#DF.
5692	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5693	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5694	*
5695	* @param pVCpu The cross context virtual CPU structure.
5696	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5697	* out-of-sync. Make sure to update the required fields
5698	* before using them.
5699	* @param pVmxTransient Pointer to the VMX transient structure.
5700	*
5701	* @remarks No-long-jump zone!!!
5702	*/
5703	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5704	{
5705	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5706
5707	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5708	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5709
5710	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5711	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5712	{
5713	uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5714	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5715
5716	typedef enum
5717	{
5718	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5719	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5720	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5721	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5722	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5723	} VMXREFLECTXCPT;
5724
5725	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5726	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5727	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5728	{
5729	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5730	{
5731	enmReflect = VMXREFLECTXCPT_XCPT;
5732	#ifdef VBOX_STRICT
5733	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5734	&& uExitVector == X86_XCPT_PF)
5735	{
5736	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5737	}
5738	#endif
5739	if ( uExitVector == X86_XCPT_PF
5740	&& uIdtVector == X86_XCPT_PF)
5741	{
5742	pVmxTransient->fVectoringDoublePF = true;
5743	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5744	}
5745	else if ( uExitVector == X86_XCPT_AC
5746	&& uIdtVector == X86_XCPT_AC)
5747	{
5748	enmReflect = VMXREFLECTXCPT_HANG;
5749	Log4(("IDT: Nested #AC - Bad guest\n"));
5750	}
5751	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5752	&& hmR0VmxIsContributoryXcpt(uExitVector)
5753	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5754	\|\| uIdtVector == X86_XCPT_PF))
5755	{
5756	enmReflect = VMXREFLECTXCPT_DF;
5757	}
5758	else if (uIdtVector == X86_XCPT_DF)
5759	enmReflect = VMXREFLECTXCPT_TF;
5760	}
5761	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5762	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5763	{
5764	/*
5765	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5766	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5767	*/
5768	enmReflect = VMXREFLECTXCPT_XCPT;
5769
5770	if (uExitVector == X86_XCPT_PF)
5771	{
5772	pVmxTransient->fVectoringPF = true;
5773	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5774	}
5775	}
5776	}
5777	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5778	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5779	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5780	{
5781	/*
5782	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5783	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5784	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5785	*/
5786	enmReflect = VMXREFLECTXCPT_XCPT;
5787	}
5788
5789	/*
5790	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5791	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5792	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5793	*
5794	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5795	*/
5796	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5797	&& enmReflect == VMXREFLECTXCPT_XCPT
5798	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5799	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5800	{
5801	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5802	}
5803
5804	switch (enmReflect)
5805	{
5806	case VMXREFLECTXCPT_XCPT:
5807	{
5808	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5809	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5810	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5811
5812	uint32_t u32ErrCode = 0;
5813	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5814	{
5815	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5816	AssertRCReturn(rc2, rc2);
5817	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5818	}
5819
5820	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5821	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5822	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5823	rcStrict = VINF_SUCCESS;
5824	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5825	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5826
5827	break;
5828	}
5829
5830	case VMXREFLECTXCPT_DF:
5831	{
5832	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5833	rcStrict = VINF_HM_DOUBLE_FAULT;
5834	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5835	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5836
5837	break;
5838	}
5839
5840	case VMXREFLECTXCPT_TF:
5841	{
5842	rcStrict = VINF_EM_RESET;
5843	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5844	uExitVector));
5845	break;
5846	}
5847
5848	case VMXREFLECTXCPT_HANG:
5849	{
5850	rcStrict = VERR_EM_GUEST_CPU_HANG;
5851	break;
5852	}
5853
5854	default:
5855	Assert(rcStrict == VINF_SUCCESS);
5856	break;
5857	}
5858	}
5859	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5860	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5861	&& uExitVector != X86_XCPT_DF
5862	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5863	{
5864	/*
5865	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5866	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5867	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5868	*/
5869	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5870	{
5871	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5872	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5873	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5874	}
5875	}
5876
5877	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
5878	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
5879	return rcStrict;
5880	}
5881
5882
5883	/**
5884	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5885	*
5886	* @returns VBox status code.
5887	* @param pVCpu The cross context virtual CPU structure.
5888	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5889	* out-of-sync. Make sure to update the required fields
5890	* before using them.
5891	*
5892	* @remarks No-long-jump zone!!!
5893	*/
5894	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5895	{
5896	NOREF(pMixedCtx);
5897
5898	/*
5899	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5900	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5901	*/
5902	VMMRZCallRing3Disable(pVCpu);
5903	HM_DISABLE_PREEMPT();
5904
5905	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5906	{
5907	uint32_t uVal = 0;
5908	uint32_t uShadow = 0;
5909	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5910	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5911	AssertRCReturn(rc, rc);
5912
5913	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5914	CPUMSetGuestCR0(pVCpu, uVal);
5915	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5916	}
5917
5918	HM_RESTORE_PREEMPT();
5919	VMMRZCallRing3Enable(pVCpu);
5920	return VINF_SUCCESS;
5921	}
5922
5923
5924	/**
5925	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5926	*
5927	* @returns VBox status code.
5928	* @param pVCpu The cross context virtual CPU structure.
5929	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5930	* out-of-sync. Make sure to update the required fields
5931	* before using them.
5932	*
5933	* @remarks No-long-jump zone!!!
5934	*/
5935	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5936	{
5937	NOREF(pMixedCtx);
5938
5939	int rc = VINF_SUCCESS;
5940	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5941	{
5942	uint32_t uVal = 0;
5943	uint32_t uShadow = 0;
5944	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5945	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5946	AssertRCReturn(rc, rc);
5947
5948	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5949	CPUMSetGuestCR4(pVCpu, uVal);
5950	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5951	}
5952	return rc;
5953	}
5954
5955
5956	/**
5957	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5958	*
5959	* @returns VBox status code.
5960	* @param pVCpu The cross context virtual CPU structure.
5961	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5962	* out-of-sync. Make sure to update the required fields
5963	* before using them.
5964	*
5965	* @remarks No-long-jump zone!!!
5966	*/
5967	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5968	{
5969	int rc = VINF_SUCCESS;
5970	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5971	{
5972	uint64_t u64Val = 0;
5973	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5974	AssertRCReturn(rc, rc);
5975
5976	pMixedCtx->rip = u64Val;
5977	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5978	}
5979	return rc;
5980	}
5981
5982
5983	/**
5984	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5985	*
5986	* @returns VBox status code.
5987	* @param pVCpu The cross context virtual CPU structure.
5988	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5989	* out-of-sync. Make sure to update the required fields
5990	* before using them.
5991	*
5992	* @remarks No-long-jump zone!!!
5993	*/
5994	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5995	{
5996	int rc = VINF_SUCCESS;
5997	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5998	{
5999	uint64_t u64Val = 0;
6000	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6001	AssertRCReturn(rc, rc);
6002
6003	pMixedCtx->rsp = u64Val;
6004	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6005	}
6006	return rc;
6007	}
6008
6009
6010	/**
6011	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6012	*
6013	* @returns VBox status code.
6014	* @param pVCpu The cross context virtual CPU structure.
6015	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6016	* out-of-sync. Make sure to update the required fields
6017	* before using them.
6018	*
6019	* @remarks No-long-jump zone!!!
6020	*/
6021	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6022	{
6023	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6024	{
6025	uint32_t uVal = 0;
6026	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6027	AssertRCReturn(rc, rc);
6028
6029	pMixedCtx->eflags.u32 = uVal;
6030	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6031	{
6032	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6033	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6034
6035	pMixedCtx->eflags.Bits.u1VM = 0;
6036	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6037	}
6038
6039	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6040	}
6041	return VINF_SUCCESS;
6042	}
6043
6044
6045	/**
6046	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6047	* guest-CPU context.
6048	*/
6049	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6050	{
6051	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6052	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6053	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6054	return rc;
6055	}
6056
6057
6058	/**
6059	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6060	* from the guest-state area in the VMCS.
6061	*
6062	* @param pVCpu The cross context virtual CPU structure.
6063	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6064	* out-of-sync. Make sure to update the required fields
6065	* before using them.
6066	*
6067	* @remarks No-long-jump zone!!!
6068	*/
6069	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6070	{
6071	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6072	{
6073	uint32_t uIntrState = 0;
6074	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6075	AssertRC(rc);
6076
6077	if (!uIntrState)
6078	{
6079	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6080	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6081
6082	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6083	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6084	}
6085	else
6086	{
6087	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6088	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6089	{
6090	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6091	AssertRC(rc);
6092	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6093	AssertRC(rc);
6094
6095	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6096	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6097	}
6098	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6099	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6100
6101	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6102	{
6103	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6104	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6105	}
6106	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6107	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6108	}
6109
6110	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6111	}
6112	}
6113
6114
6115	/**
6116	* Saves the guest's activity state.
6117	*
6118	* @returns VBox status code.
6119	* @param pVCpu The cross context virtual CPU structure.
6120	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6121	* out-of-sync. Make sure to update the required fields
6122	* before using them.
6123	*
6124	* @remarks No-long-jump zone!!!
6125	*/
6126	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6127	{
6128	NOREF(pMixedCtx);
6129	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6130	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6131	return VINF_SUCCESS;
6132	}
6133
6134
6135	/**
6136	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6137	* the current VMCS into the guest-CPU context.
6138	*
6139	* @returns VBox status code.
6140	* @param pVCpu The cross context virtual CPU structure.
6141	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6142	* out-of-sync. Make sure to update the required fields
6143	* before using them.
6144	*
6145	* @remarks No-long-jump zone!!!
6146	*/
6147	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6148	{
6149	int rc = VINF_SUCCESS;
6150	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6151	{
6152	uint32_t u32Val = 0;
6153	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6154	pMixedCtx->SysEnter.cs = u32Val;
6155	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6156	}
6157
6158	uint64_t u64Val = 0;
6159	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6160	{
6161	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6162	pMixedCtx->SysEnter.eip = u64Val;
6163	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6164	}
6165	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6166	{
6167	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6168	pMixedCtx->SysEnter.esp = u64Val;
6169	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6170	}
6171	return rc;
6172	}
6173
6174
6175	/**
6176	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6177	* the CPU back into the guest-CPU context.
6178	*
6179	* @returns VBox status code.
6180	* @param pVCpu The cross context virtual CPU structure.
6181	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6182	* out-of-sync. Make sure to update the required fields
6183	* before using them.
6184	*
6185	* @remarks No-long-jump zone!!!
6186	*/
6187	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6188	{
6189	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6190	VMMRZCallRing3Disable(pVCpu);
6191	HM_DISABLE_PREEMPT();
6192
6193	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6194	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6195	{
6196	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6197	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6198	}
6199
6200	HM_RESTORE_PREEMPT();
6201	VMMRZCallRing3Enable(pVCpu);
6202
6203	return VINF_SUCCESS;
6204	}
6205
6206
6207	/**
6208	* Saves the auto load/store'd guest MSRs from the current VMCS into
6209	* the guest-CPU context.
6210	*
6211	* @returns VBox status code.
6212	* @param pVCpu The cross context virtual CPU structure.
6213	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6214	* out-of-sync. Make sure to update the required fields
6215	* before using them.
6216	*
6217	* @remarks No-long-jump zone!!!
6218	*/
6219	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6220	{
6221	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6222	return VINF_SUCCESS;
6223
6224	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6225	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6226	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6227	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6228	{
6229	switch (pMsr->u32Msr)
6230	{
6231	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6232	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6233	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6234	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6235	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6236	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6237	break;
6238
6239	default:
6240	{
6241	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6242	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6243	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6244	}
6245	}
6246	}
6247
6248	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6249	return VINF_SUCCESS;
6250	}
6251
6252
6253	/**
6254	* Saves the guest control registers from the current VMCS into the guest-CPU
6255	* context.
6256	*
6257	* @returns VBox status code.
6258	* @param pVCpu The cross context virtual CPU structure.
6259	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6260	* out-of-sync. Make sure to update the required fields
6261	* before using them.
6262	*
6263	* @remarks No-long-jump zone!!!
6264	*/
6265	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6266	{
6267	/* Guest CR0. Guest FPU. */
6268	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6269	AssertRCReturn(rc, rc);
6270
6271	/* Guest CR4. */
6272	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6273	AssertRCReturn(rc, rc);
6274
6275	/* Guest CR2 - updated always during the world-switch or in #PF. */
6276	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6277	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6278	{
6279	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6280	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6281
6282	PVM pVM = pVCpu->CTX_SUFF(pVM);
6283	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6284	\|\| ( pVM->hm.s.fNestedPaging
6285	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6286	{
6287	uint64_t u64Val = 0;
6288	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6289	if (pMixedCtx->cr3 != u64Val)
6290	{
6291	CPUMSetGuestCR3(pVCpu, u64Val);
6292	if (VMMRZCallRing3IsEnabled(pVCpu))
6293	{
6294	PGMUpdateCR3(pVCpu, u64Val);
6295	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6296	}
6297	else
6298	{
6299	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6300	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6301	}
6302	}
6303
6304	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6305	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6306	{
6307	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6308	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6309	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6310	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6311	AssertRCReturn(rc, rc);
6312
6313	if (VMMRZCallRing3IsEnabled(pVCpu))
6314	{
6315	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6316	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6317	}
6318	else
6319	{
6320	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6321	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6322	}
6323	}
6324	}
6325
6326	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6327	}
6328
6329	/*
6330	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6331	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6332	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6333	*
6334	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6335	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6336	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6337	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6338	*
6339	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6340	*/
6341	if (VMMRZCallRing3IsEnabled(pVCpu))
6342	{
6343	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6344	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6345
6346	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6347	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6348
6349	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6350	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6351	}
6352
6353	return rc;
6354	}
6355
6356
6357	/**
6358	* Reads a guest segment register from the current VMCS into the guest-CPU
6359	* context.
6360	*
6361	* @returns VBox status code.
6362	* @param pVCpu The cross context virtual CPU structure.
6363	* @param idxSel Index of the selector in the VMCS.
6364	* @param idxLimit Index of the segment limit in the VMCS.
6365	* @param idxBase Index of the segment base in the VMCS.
6366	* @param idxAccess Index of the access rights of the segment in the VMCS.
6367	* @param pSelReg Pointer to the segment selector.
6368	*
6369	* @remarks No-long-jump zone!!!
6370	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6371	* macro as that takes care of whether to read from the VMCS cache or
6372	* not.
6373	*/
6374	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6375	PCPUMSELREG pSelReg)
6376	{
6377	NOREF(pVCpu);
6378
6379	uint32_t u32Val = 0;
6380	int rc = VMXReadVmcs32(idxSel, &u32Val);
6381	AssertRCReturn(rc, rc);
6382	pSelReg->Sel = (uint16_t)u32Val;
6383	pSelReg->ValidSel = (uint16_t)u32Val;
6384	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6385
6386	rc = VMXReadVmcs32(idxLimit, &u32Val);
6387	AssertRCReturn(rc, rc);
6388	pSelReg->u32Limit = u32Val;
6389
6390	uint64_t u64Val = 0;
6391	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6392	AssertRCReturn(rc, rc);
6393	pSelReg->u64Base = u64Val;
6394
6395	rc = VMXReadVmcs32(idxAccess, &u32Val);
6396	AssertRCReturn(rc, rc);
6397	pSelReg->Attr.u = u32Val;
6398
6399	/*
6400	* If VT-x marks the segment as unusable, most other bits remain undefined:
6401	* - For CS the L, D and G bits have meaning.
6402	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6403	* - For the remaining data segments no bits are defined.
6404	*
6405	* The present bit and the unusable bit has been observed to be set at the
6406	* same time (the selector was supposed to be invalid as we started executing
6407	* a V8086 interrupt in ring-0).
6408	*
6409	* What should be important for the rest of the VBox code, is that the P bit is
6410	* cleared. Some of the other VBox code recognizes the unusable bit, but
6411	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6412	* safe side here, we'll strip off P and other bits we don't care about. If
6413	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6414	*
6415	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6416	*/
6417	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6418	{
6419	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6420
6421	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6422	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6423	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6424
6425	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6426	#ifdef DEBUG_bird
6427	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6428	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6429	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6430	#endif
6431	}
6432	return VINF_SUCCESS;
6433	}
6434
6435
6436	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6437	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6438	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6439	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6440	#else
6441	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6442	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6443	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6444	#endif
6445
6446
6447	/**
6448	* Saves the guest segment registers from the current VMCS into the guest-CPU
6449	* context.
6450	*
6451	* @returns VBox status code.
6452	* @param pVCpu The cross context virtual CPU structure.
6453	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6454	* out-of-sync. Make sure to update the required fields
6455	* before using them.
6456	*
6457	* @remarks No-long-jump zone!!!
6458	*/
6459	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6460	{
6461	/* Guest segment registers. */
6462	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6463	{
6464	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6465	AssertRCReturn(rc, rc);
6466
6467	rc = VMXLOCAL_READ_SEG(CS, cs);
6468	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6469	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6470	rc \|= VMXLOCAL_READ_SEG(ES, es);
6471	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6472	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6473	AssertRCReturn(rc, rc);
6474
6475	/* Restore segment attributes for real-on-v86 mode hack. */
6476	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6477	{
6478	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6479	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6480	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6481	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6482	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6483	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6484	}
6485	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6486	}
6487
6488	return VINF_SUCCESS;
6489	}
6490
6491
6492	/**
6493	* Saves the guest descriptor table registers and task register from the current
6494	* VMCS into the guest-CPU context.
6495	*
6496	* @returns VBox status code.
6497	* @param pVCpu The cross context virtual CPU structure.
6498	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6499	* out-of-sync. Make sure to update the required fields
6500	* before using them.
6501	*
6502	* @remarks No-long-jump zone!!!
6503	*/
6504	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6505	{
6506	int rc = VINF_SUCCESS;
6507
6508	/* Guest LDTR. */
6509	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6510	{
6511	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6512	AssertRCReturn(rc, rc);
6513	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6514	}
6515
6516	/* Guest GDTR. */
6517	uint64_t u64Val = 0;
6518	uint32_t u32Val = 0;
6519	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6520	{
6521	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6522	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6523	pMixedCtx->gdtr.pGdt = u64Val;
6524	pMixedCtx->gdtr.cbGdt = u32Val;
6525	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6526	}
6527
6528	/* Guest IDTR. */
6529	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6530	{
6531	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6532	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6533	pMixedCtx->idtr.pIdt = u64Val;
6534	pMixedCtx->idtr.cbIdt = u32Val;
6535	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6536	}
6537
6538	/* Guest TR. */
6539	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6540	{
6541	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6542	AssertRCReturn(rc, rc);
6543
6544	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6545	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6546	{
6547	rc = VMXLOCAL_READ_SEG(TR, tr);
6548	AssertRCReturn(rc, rc);
6549	}
6550	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6551	}
6552	return rc;
6553	}
6554
6555	#undef VMXLOCAL_READ_SEG
6556
6557
6558	/**
6559	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6560	* context.
6561	*
6562	* @returns VBox status code.
6563	* @param pVCpu The cross context virtual CPU structure.
6564	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6565	* out-of-sync. Make sure to update the required fields
6566	* before using them.
6567	*
6568	* @remarks No-long-jump zone!!!
6569	*/
6570	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6571	{
6572	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6573	{
6574	if (!pVCpu->hm.s.fUsingHyperDR7)
6575	{
6576	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6577	uint32_t u32Val;
6578	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6579	pMixedCtx->dr[7] = u32Val;
6580	}
6581
6582	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6583	}
6584	return VINF_SUCCESS;
6585	}
6586
6587
6588	/**
6589	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6590	*
6591	* @returns VBox status code.
6592	* @param pVCpu The cross context virtual CPU structure.
6593	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6594	* out-of-sync. Make sure to update the required fields
6595	* before using them.
6596	*
6597	* @remarks No-long-jump zone!!!
6598	*/
6599	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6600	{
6601	NOREF(pMixedCtx);
6602
6603	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6604	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6605	return VINF_SUCCESS;
6606	}
6607
6608
6609	/**
6610	* Saves the entire guest state from the currently active VMCS into the
6611	* guest-CPU context.
6612	*
6613	* This essentially VMREADs all guest-data.
6614	*
6615	* @returns VBox status code.
6616	* @param pVCpu The cross context virtual CPU structure.
6617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6618	* out-of-sync. Make sure to update the required fields
6619	* before using them.
6620	*/
6621	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6622	{
6623	Assert(pVCpu);
6624	Assert(pMixedCtx);
6625
6626	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6627	return VINF_SUCCESS;
6628
6629	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6630	again on the ring-3 callback path, there is no real need to. */
6631	if (VMMRZCallRing3IsEnabled(pVCpu))
6632	VMMR0LogFlushDisable(pVCpu);
6633	else
6634	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6635	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6636
6637	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6638	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6639
6640	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6641	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6642
6643	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6644	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6645
6646	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6647	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6648
6649	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6650	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6651
6652	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6653	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6654
6655	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6656	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6657
6658	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6659	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6660
6661	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6662	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6663
6664	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6665	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6666
6667	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6668	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6669	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6670
6671	if (VMMRZCallRing3IsEnabled(pVCpu))
6672	VMMR0LogFlushEnable(pVCpu);
6673
6674	return VINF_SUCCESS;
6675	}
6676
6677
6678	/**
6679	* Saves basic guest registers needed for IEM instruction execution.
6680	*
6681	* @returns VBox status code (OR-able).
6682	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
6683	* @param pMixedCtx Pointer to the CPU context of the guest.
6684	* @param fMemory Whether the instruction being executed operates on
6685	* memory or not. Only CR0 is synced up if clear.
6686	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6687	*/
6688	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6689	{
6690	/*
6691	* We assume all general purpose registers other than RSP are available.
6692	*
6693	* RIP is a must, as it will be incremented or otherwise changed.
6694	*
6695	* RFLAGS are always required to figure the CPL.
6696	*
6697	* RSP isn't always required, however it's a GPR, so frequently required.
6698	*
6699	* SS and CS are the only segment register needed if IEM doesn't do memory
6700	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6701	*
6702	* CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6703	* be required for memory accesses.
6704	*
6705	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
6706	*/
6707	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6708	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6709	if (fNeedRsp)
6710	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6711	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6712	if (!fMemory)
6713	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6714	else
6715	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6716	AssertRCReturn(rc, rc);
6717	return rc;
6718	}
6719
6720
6721	/**
6722	* Ensures that we've got a complete basic guest-context.
6723	*
6724	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
6725	* is for the interpreter.
6726	*
6727	* @returns VBox status code.
6728	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
6729	* @param pMixedCtx Pointer to the guest-CPU context which may have data
6730	* needing to be synced in.
6731	* @thread EMT(pVCpu)
6732	*/
6733	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6734	{
6735	/* Note! Since this is only applicable to VT-x, the implementation is placed
6736	in the VT-x part of the sources instead of the generic stuff. */
6737	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6738	{
6739	/* For now, imply that the caller might change everything too. */
6740	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6741	return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6742	}
6743	return VINF_SUCCESS;
6744	}
6745
6746
6747	/**
6748	* Check per-VM and per-VCPU force flag actions that require us to go back to
6749	* ring-3 for one reason or another.
6750	*
6751	* @returns Strict VBox status code (i.e. informational status codes too)
6752	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6753	* ring-3.
6754	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6755	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6756	* interrupts)
6757	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6758	* all EMTs to be in ring-3.
6759	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6760	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6761	* to the EM loop.
6762	*
6763	* @param pVM The cross context VM structure.
6764	* @param pVCpu The cross context virtual CPU structure.
6765	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6766	* out-of-sync. Make sure to update the required fields
6767	* before using them.
6768	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
6769	*/
6770	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6771	{
6772	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6773
6774	/*
6775	* Anything pending? Should be more likely than not if we're doing a good job.
6776	*/
6777	if ( !fStepping
6778	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6779	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6780	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6781	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6782	return VINF_SUCCESS;
6783
6784	/* We need the control registers now, make sure the guest-CPU context is updated. */
6785	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6786	AssertRCReturn(rc3, rc3);
6787
6788	/* Pending HM CR3 sync. */
6789	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6790	{
6791	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6792	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6793	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6794	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6795	}
6796
6797	/* Pending HM PAE PDPEs. */
6798	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6799	{
6800	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6801	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6802	}
6803
6804	/* Pending PGM C3 sync. */
6805	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6806	{
6807	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6808	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6809	if (rcStrict2 != VINF_SUCCESS)
6810	{
6811	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6812	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6813	return rcStrict2;
6814	}
6815	}
6816
6817	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6818	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6819	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6820	{
6821	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6822	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6823	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6824	return rc2;
6825	}
6826
6827	/* Pending VM request packets, such as hardware interrupts. */
6828	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6829	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6830	{
6831	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6832	return VINF_EM_PENDING_REQUEST;
6833	}
6834
6835	/* Pending PGM pool flushes. */
6836	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6837	{
6838	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6839	return VINF_PGM_POOL_FLUSH_PENDING;
6840	}
6841
6842	/* Pending DMA requests. */
6843	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6844	{
6845	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6846	return VINF_EM_RAW_TO_R3;
6847	}
6848
6849	return VINF_SUCCESS;
6850	}
6851
6852
6853	/**
6854	* Converts any TRPM trap into a pending HM event. This is typically used when
6855	* entering from ring-3 (not longjmp returns).
6856	*
6857	* @param pVCpu The cross context virtual CPU structure.
6858	*/
6859	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6860	{
6861	Assert(TRPMHasTrap(pVCpu));
6862	Assert(!pVCpu->hm.s.Event.fPending);
6863
6864	uint8_t uVector;
6865	TRPMEVENT enmTrpmEvent;
6866	RTGCUINT uErrCode;
6867	RTGCUINTPTR GCPtrFaultAddress;
6868	uint8_t cbInstr;
6869
6870	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6871	AssertRC(rc);
6872
6873	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6874	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6875	if (enmTrpmEvent == TRPM_TRAP)
6876	{
6877	switch (uVector)
6878	{
6879	case X86_XCPT_NMI:
6880	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6881	break;
6882
6883	case X86_XCPT_BP:
6884	case X86_XCPT_OF:
6885	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6886	break;
6887
6888	case X86_XCPT_PF:
6889	case X86_XCPT_DF:
6890	case X86_XCPT_TS:
6891	case X86_XCPT_NP:
6892	case X86_XCPT_SS:
6893	case X86_XCPT_GP:
6894	case X86_XCPT_AC:
6895	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6896	/* no break! */
6897	default:
6898	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6899	break;
6900	}
6901	}
6902	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6903	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6904	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6905	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6906	else
6907	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6908
6909	rc = TRPMResetTrap(pVCpu);
6910	AssertRC(rc);
6911	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6912	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6913
6914	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6915	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6916	}
6917
6918
6919	/**
6920	* Converts the pending HM event into a TRPM trap.
6921	*
6922	* @param pVCpu The cross context virtual CPU structure.
6923	*/
6924	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6925	{
6926	Assert(pVCpu->hm.s.Event.fPending);
6927
6928	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6929	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6930	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6931	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6932
6933	/* If a trap was already pending, we did something wrong! */
6934	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6935
6936	TRPMEVENT enmTrapType;
6937	switch (uVectorType)
6938	{
6939	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6940	enmTrapType = TRPM_HARDWARE_INT;
6941	break;
6942
6943	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6944	enmTrapType = TRPM_SOFTWARE_INT;
6945	break;
6946
6947	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6948	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6949	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6950	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6951	enmTrapType = TRPM_TRAP;
6952	break;
6953
6954	default:
6955	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6956	enmTrapType = TRPM_32BIT_HACK;
6957	break;
6958	}
6959
6960	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6961
6962	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6963	AssertRC(rc);
6964
6965	if (fErrorCodeValid)
6966	TRPMSetErrorCode(pVCpu, uErrorCode);
6967
6968	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6969	&& uVector == X86_XCPT_PF)
6970	{
6971	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6972	}
6973	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6974	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6975	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6976	{
6977	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6978	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6979	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6980	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6981	}
6982
6983	/* Clear any pending events from the VMCS. */
6984	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6985	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6986
6987	/* We're now done converting the pending event. */
6988	pVCpu->hm.s.Event.fPending = false;
6989	}
6990
6991
6992	/**
6993	* Does the necessary state syncing before returning to ring-3 for any reason
6994	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6995	*
6996	* @returns VBox status code.
6997	* @param pVM The cross context VM structure.
6998	* @param pVCpu The cross context virtual CPU structure.
6999	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7000	* be out-of-sync. Make sure to update the required
7001	* fields before using them.
7002	* @param fSaveGuestState Whether to save the guest state or not.
7003	*
7004	* @remarks No-long-jmp zone!!!
7005	*/
7006	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7007	{
7008	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7009	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7010
7011	RTCPUID idCpu = RTMpCpuId();
7012	Log4Func(("HostCpuId=%u\n", idCpu));
7013
7014	/*
7015	* !!! IMPORTANT !!!
7016	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7017	*/
7018
7019	/* Save the guest state if necessary. */
7020	if ( fSaveGuestState
7021	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7022	{
7023	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7024	AssertRCReturn(rc, rc);
7025	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7026	}
7027
7028	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7029	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7030	{
7031	if (fSaveGuestState)
7032	{
7033	/* We shouldn't reload CR0 without saving it first. */
7034	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7035	AssertRCReturn(rc, rc);
7036	}
7037	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7038	}
7039
7040	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7041	#ifdef VBOX_STRICT
7042	if (CPUMIsHyperDebugStateActive(pVCpu))
7043	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7044	#endif
7045	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7046	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7047	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7048	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7049
7050	#if HC_ARCH_BITS == 64
7051	/* Restore host-state bits that VT-x only restores partially. */
7052	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7053	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7054	{
7055	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7056	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7057	}
7058	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7059	#endif
7060
7061	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7062	if (pVCpu->hm.s.vmx.fLazyMsrs)
7063	{
7064	/* We shouldn't reload the guest MSRs without saving it first. */
7065	if (!fSaveGuestState)
7066	{
7067	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7068	AssertRCReturn(rc, rc);
7069	}
7070	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7071	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7072	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7073	}
7074
7075	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7076	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7077
7078	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7079	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7080	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7081	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7082	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7083	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7084	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7085	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7086
7087	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7088
7089	/** @todo This partially defeats the purpose of having preemption hooks.
7090	* The problem is, deregistering the hooks should be moved to a place that
7091	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7092	* context.
7093	*/
7094	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7095	{
7096	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7097	AssertRCReturn(rc, rc);
7098
7099	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7100	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7101	}
7102	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7103	NOREF(idCpu);
7104
7105	return VINF_SUCCESS;
7106	}
7107
7108
7109	/**
7110	* Leaves the VT-x session.
7111	*
7112	* @returns VBox status code.
7113	* @param pVM The cross context VM structure.
7114	* @param pVCpu The cross context virtual CPU structure.
7115	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7116	* out-of-sync. Make sure to update the required fields
7117	* before using them.
7118	*
7119	* @remarks No-long-jmp zone!!!
7120	*/
7121	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7122	{
7123	HM_DISABLE_PREEMPT();
7124	HMVMX_ASSERT_CPU_SAFE();
7125	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7126	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7127
7128	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7129	and done this from the VMXR0ThreadCtxCallback(). */
7130	if (!pVCpu->hm.s.fLeaveDone)
7131	{
7132	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7133	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7134	pVCpu->hm.s.fLeaveDone = true;
7135	}
7136	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7137
7138	/*
7139	* !!! IMPORTANT !!!
7140	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7141	*/
7142
7143	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7144	/** @todo Deregistering here means we need to VMCLEAR always
7145	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7146	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7147	VMMR0ThreadCtxHookDisable(pVCpu);
7148
7149	/* Leave HM context. This takes care of local init (term). */
7150	int rc = HMR0LeaveCpu(pVCpu);
7151
7152	HM_RESTORE_PREEMPT();
7153	return rc;
7154	}
7155
7156
7157	/**
7158	* Does the necessary state syncing before doing a longjmp to ring-3.
7159	*
7160	* @returns VBox status code.
7161	* @param pVM The cross context VM structure.
7162	* @param pVCpu The cross context virtual CPU structure.
7163	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7164	* out-of-sync. Make sure to update the required fields
7165	* before using them.
7166	*
7167	* @remarks No-long-jmp zone!!!
7168	*/
7169	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7170	{
7171	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7172	}
7173
7174
7175	/**
7176	* Take necessary actions before going back to ring-3.
7177	*
7178	* An action requires us to go back to ring-3. This function does the necessary
7179	* steps before we can safely return to ring-3. This is not the same as longjmps
7180	* to ring-3, this is voluntary and prepares the guest so it may continue
7181	* executing outside HM (recompiler/IEM).
7182	*
7183	* @returns VBox status code.
7184	* @param pVM The cross context VM structure.
7185	* @param pVCpu The cross context virtual CPU structure.
7186	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7187	* out-of-sync. Make sure to update the required fields
7188	* before using them.
7189	* @param rcExit The reason for exiting to ring-3. Can be
7190	* VINF_VMM_UNKNOWN_RING3_CALL.
7191	*/
7192	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7193	{
7194	Assert(pVM);
7195	Assert(pVCpu);
7196	Assert(pMixedCtx);
7197	HMVMX_ASSERT_PREEMPT_SAFE();
7198
7199	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7200	{
7201	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7202	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7203	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7204	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7205	}
7206
7207	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7208	VMMRZCallRing3Disable(pVCpu);
7209	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7210
7211	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7212	if (pVCpu->hm.s.Event.fPending)
7213	{
7214	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7215	Assert(!pVCpu->hm.s.Event.fPending);
7216	}
7217
7218	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7219	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7220
7221	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7222	and if we're injecting an event we should have a TRPM trap pending. */
7223	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7224	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7225	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7226	#endif
7227
7228	/* Save guest state and restore host state bits. */
7229	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7230	AssertRCReturn(rc, rc);
7231	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7232	/* Thread-context hooks are unregistered at this point!!! */
7233
7234	/* Sync recompiler state. */
7235	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7236	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7237	\| CPUM_CHANGED_LDTR
7238	\| CPUM_CHANGED_GDTR
7239	\| CPUM_CHANGED_IDTR
7240	\| CPUM_CHANGED_TR
7241	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7242	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7243	if ( pVM->hm.s.fNestedPaging
7244	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7245	{
7246	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7247	}
7248
7249	Assert(!pVCpu->hm.s.fClearTrapFlag);
7250
7251	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7252	if (rcExit != VINF_EM_RAW_INTERRUPT)
7253	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7254
7255	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7256
7257	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7258	VMMRZCallRing3RemoveNotification(pVCpu);
7259	VMMRZCallRing3Enable(pVCpu);
7260
7261	return rc;
7262	}
7263
7264
7265	/**
7266	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7267	* longjump to ring-3 and possibly get preempted.
7268	*
7269	* @returns VBox status code.
7270	* @param pVCpu The cross context virtual CPU structure.
7271	* @param enmOperation The operation causing the ring-3 longjump.
7272	* @param pvUser Opaque pointer to the guest-CPU context. The data
7273	* may be out-of-sync. Make sure to update the required
7274	* fields before using them.
7275	*/
7276	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7277	{
7278	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7279	{
7280	/*
7281	* !!! IMPORTANT !!!
7282	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7283	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7284	*/
7285	VMMRZCallRing3RemoveNotification(pVCpu);
7286	VMMRZCallRing3Disable(pVCpu);
7287	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7288	RTThreadPreemptDisable(&PreemptState);
7289
7290	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7291	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7292
7293	#if HC_ARCH_BITS == 64
7294	/* Restore host-state bits that VT-x only restores partially. */
7295	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7296	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7297	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7298	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7299	#endif
7300	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7301	if (pVCpu->hm.s.vmx.fLazyMsrs)
7302	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7303
7304	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7305	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7306	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7307	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7308	{
7309	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7310	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7311	}
7312
7313	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7314	VMMR0ThreadCtxHookDisable(pVCpu);
7315	HMR0LeaveCpu(pVCpu);
7316	RTThreadPreemptRestore(&PreemptState);
7317	return VINF_SUCCESS;
7318	}
7319
7320	Assert(pVCpu);
7321	Assert(pvUser);
7322	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7323	HMVMX_ASSERT_PREEMPT_SAFE();
7324
7325	VMMRZCallRing3Disable(pVCpu);
7326	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7327
7328	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7329	enmOperation));
7330
7331	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7332	AssertRCReturn(rc, rc);
7333
7334	VMMRZCallRing3Enable(pVCpu);
7335	return VINF_SUCCESS;
7336	}
7337
7338
7339	/**
7340	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7341	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7342	*
7343	* @param pVCpu The cross context virtual CPU structure.
7344	*/
7345	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7346	{
7347	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7348	{
7349	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7350	{
7351	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7352	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7353	AssertRC(rc);
7354	Log4(("Setup interrupt-window exiting\n"));
7355	}
7356	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7357	}
7358
7359
7360	/**
7361	* Clears the interrupt-window exiting control in the VMCS.
7362	*
7363	* @param pVCpu The cross context virtual CPU structure.
7364	*/
7365	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7366	{
7367	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7368	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7369	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7370	AssertRC(rc);
7371	Log4(("Cleared interrupt-window exiting\n"));
7372	}
7373
7374
7375	/**
7376	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7377	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7378	*
7379	* @param pVCpu The cross context virtual CPU structure.
7380	*/
7381	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7382	{
7383	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7384	{
7385	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7386	{
7387	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7388	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7389	AssertRC(rc);
7390	Log4(("Setup NMI-window exiting\n"));
7391	}
7392	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7393	}
7394
7395
7396	/**
7397	* Clears the NMI-window exiting control in the VMCS.
7398	*
7399	* @param pVCpu The cross context virtual CPU structure.
7400	*/
7401	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7402	{
7403	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7404	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7405	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7406	AssertRC(rc);
7407	Log4(("Cleared NMI-window exiting\n"));
7408	}
7409
7410
7411	/**
7412	* Evaluates the event to be delivered to the guest and sets it as the pending
7413	* event.
7414	*
7415	* @returns The VT-x guest-interruptibility state.
7416	* @param pVCpu The cross context virtual CPU structure.
7417	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7418	* out-of-sync. Make sure to update the required fields
7419	* before using them.
7420	*/
7421	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7422	{
7423	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7424	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7425	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7426	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7427	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7428
7429	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7430	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7431	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7432	Assert(!TRPMHasTrap(pVCpu));
7433
7434	#ifdef VBOX_WITH_NEW_APIC
7435	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7436	APICUpdatePendingInterrupts(pVCpu);
7437	#endif
7438
7439	/*
7440	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7441	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7442	*/
7443	/** @todo SMI. SMIs take priority over NMIs. */
7444	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7445	{
7446	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7447	if ( !pVCpu->hm.s.Event.fPending
7448	&& !fBlockNmi
7449	&& !fBlockSti
7450	&& !fBlockMovSS)
7451	{
7452	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7453	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7454	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7455
7456	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7457	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7458	}
7459	else
7460	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7461	}
7462	/*
7463	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7464	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7465	*/
7466	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7467	&& !pVCpu->hm.s.fSingleInstruction)
7468	{
7469	Assert(!DBGFIsStepping(pVCpu));
7470	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7471	AssertRC(rc);
7472	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7473	if ( !pVCpu->hm.s.Event.fPending
7474	&& !fBlockInt
7475	&& !fBlockSti
7476	&& !fBlockMovSS)
7477	{
7478	uint8_t u8Interrupt;
7479	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7480	if (RT_SUCCESS(rc))
7481	{
7482	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7483	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7484	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7485
7486	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7487	}
7488	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7489	{
7490	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7491	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7492	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7493	}
7494	else
7495	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7496	}
7497	else
7498	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7499	}
7500
7501	return uIntrState;
7502	}
7503
7504
7505	/**
7506	* Sets a pending-debug exception to be delivered to the guest if the guest is
7507	* single-stepping in the VMCS.
7508	*
7509	* @param pVCpu The cross context virtual CPU structure.
7510	*/
7511	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7512	{
7513	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7514	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7515	AssertRC(rc);
7516	}
7517
7518
7519	/**
7520	* Injects any pending events into the guest if the guest is in a state to
7521	* receive them.
7522	*
7523	* @returns Strict VBox status code (i.e. informational status codes too).
7524	* @param pVCpu The cross context virtual CPU structure.
7525	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7526	* out-of-sync. Make sure to update the required fields
7527	* before using them.
7528	* @param uIntrState The VT-x guest-interruptibility state.
7529	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7530	* return VINF_EM_DBG_STEPPED if the event was
7531	* dispatched directly.
7532	*/
7533	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7534	{
7535	HMVMX_ASSERT_PREEMPT_SAFE();
7536	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7537
7538	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7539	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7540
7541	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7542	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7543	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7544	Assert(!TRPMHasTrap(pVCpu));
7545
7546	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7547	if (pVCpu->hm.s.Event.fPending)
7548	{
7549	/*
7550	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7551	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7552	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7553	*
7554	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7555	*/
7556	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7557	#ifdef VBOX_STRICT
7558	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7559	{
7560	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7561	Assert(!fBlockInt);
7562	Assert(!fBlockSti);
7563	Assert(!fBlockMovSS);
7564	}
7565	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7566	{
7567	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7568	Assert(!fBlockSti);
7569	Assert(!fBlockMovSS);
7570	Assert(!fBlockNmi);
7571	}
7572	#endif
7573	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7574	(uint8_t)uIntType));
7575	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7576	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7577	fStepping, &uIntrState);
7578	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7579
7580	/* Update the interruptibility-state as it could have been changed by
7581	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7582	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7583	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7584
7585	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7586	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7587	else
7588	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7589	}
7590
7591	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7592	if ( fBlockSti
7593	\|\| fBlockMovSS)
7594	{
7595	if (!pVCpu->hm.s.fSingleInstruction)
7596	{
7597	/*
7598	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7599	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7600	* See Intel spec. 27.3.4 "Saving Non-Register State".
7601	*/
7602	Assert(!DBGFIsStepping(pVCpu));
7603	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7604	AssertRCReturn(rc2, rc2);
7605	if (pMixedCtx->eflags.Bits.u1TF)
7606	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7607	}
7608	else if (pMixedCtx->eflags.Bits.u1TF)
7609	{
7610	/*
7611	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7612	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7613	*/
7614	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7615	uIntrState = 0;
7616	}
7617	}
7618
7619	/*
7620	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7621	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7622	*/
7623	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7624	AssertRC(rc2);
7625
7626	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7627	NOREF(fBlockMovSS); NOREF(fBlockSti);
7628	return rcStrict;
7629	}
7630
7631
7632	/**
7633	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7634	*
7635	* @param pVCpu The cross context virtual CPU structure.
7636	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7637	* out-of-sync. Make sure to update the required fields
7638	* before using them.
7639	*/
7640	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7641	{
7642	NOREF(pMixedCtx);
7643	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7644	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7645	}
7646
7647
7648	/**
7649	* Injects a double-fault (\#DF) exception into the VM.
7650	*
7651	* @returns Strict VBox status code (i.e. informational status codes too).
7652	* @param pVCpu The cross context virtual CPU structure.
7653	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7654	* out-of-sync. Make sure to update the required fields
7655	* before using them.
7656	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7657	* and should return VINF_EM_DBG_STEPPED if the event
7658	* is injected directly (register modified by us, not
7659	* by hardware on VM-entry).
7660	* @param puIntrState Pointer to the current guest interruptibility-state.
7661	* This interruptibility-state will be updated if
7662	* necessary. This cannot not be NULL.
7663	*/
7664	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7665	{
7666	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7667	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7668	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7669	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7670	fStepping, puIntrState);
7671	}
7672
7673
7674	/**
7675	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7676	*
7677	* @param pVCpu The cross context virtual CPU structure.
7678	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7679	* out-of-sync. Make sure to update the required fields
7680	* before using them.
7681	*/
7682	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7683	{
7684	NOREF(pMixedCtx);
7685	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7686	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7687	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7688	}
7689
7690
7691	/**
7692	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7693	*
7694	* @param pVCpu The cross context virtual CPU structure.
7695	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7696	* out-of-sync. Make sure to update the required fields
7697	* before using them.
7698	* @param cbInstr The value of RIP that is to be pushed on the guest
7699	* stack.
7700	*/
7701	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7702	{
7703	NOREF(pMixedCtx);
7704	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7705	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7706	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7707	}
7708
7709
7710	/**
7711	* Injects a general-protection (\#GP) fault into the VM.
7712	*
7713	* @returns Strict VBox status code (i.e. informational status codes too).
7714	* @param pVCpu The cross context virtual CPU structure.
7715	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7716	* out-of-sync. Make sure to update the required fields
7717	* before using them.
7718	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7719	* mode, i.e. in real-mode it's not valid).
7720	* @param u32ErrorCode The error code associated with the \#GP.
7721	* @param fStepping Whether we're running in
7722	* hmR0VmxRunGuestCodeStep() and should return
7723	* VINF_EM_DBG_STEPPED if the event is injected
7724	* directly (register modified by us, not by
7725	* hardware on VM-entry).
7726	* @param puIntrState Pointer to the current guest interruptibility-state.
7727	* This interruptibility-state will be updated if
7728	* necessary. This cannot not be NULL.
7729	*/
7730	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7731	bool fStepping, uint32_t *puIntrState)
7732	{
7733	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7734	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7735	if (fErrorCodeValid)
7736	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7737	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7738	fStepping, puIntrState);
7739	}
7740
7741
7742	/**
7743	* Sets a general-protection (\#GP) exception as pending-for-injection into the
7744	* VM.
7745	*
7746	* @param pVCpu The cross context virtual CPU structure.
7747	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7748	* out-of-sync. Make sure to update the required fields
7749	* before using them.
7750	* @param u32ErrorCode The error code associated with the \#GP.
7751	*/
7752	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7753	{
7754	NOREF(pMixedCtx);
7755	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7756	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7757	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7758	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
7759	}
7760
7761
7762	/**
7763	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7764	*
7765	* @param pVCpu The cross context virtual CPU structure.
7766	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7767	* out-of-sync. Make sure to update the required fields
7768	* before using them.
7769	* @param uVector The software interrupt vector number.
7770	* @param cbInstr The value of RIP that is to be pushed on the guest
7771	* stack.
7772	*/
7773	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7774	{
7775	NOREF(pMixedCtx);
7776	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7777	if ( uVector == X86_XCPT_BP
7778	\|\| uVector == X86_XCPT_OF)
7779	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7780	else
7781	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7782	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7783	}
7784
7785
7786	/**
7787	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7788	* stack.
7789	*
7790	* @returns Strict VBox status code (i.e. informational status codes too).
7791	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7792	* @param pVM The cross context VM structure.
7793	* @param pMixedCtx Pointer to the guest-CPU context.
7794	* @param uValue The value to push to the guest stack.
7795	*/
7796	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7797	{
7798	/*
7799	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7800	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7801	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7802	*/
7803	if (pMixedCtx->sp == 1)
7804	return VINF_EM_RESET;
7805	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7806	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7807	AssertRC(rc);
7808	return rc;
7809	}
7810
7811
7812	/**
7813	* Injects an event into the guest upon VM-entry by updating the relevant fields
7814	* in the VM-entry area in the VMCS.
7815	*
7816	* @returns Strict VBox status code (i.e. informational status codes too).
7817	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7818	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7819	*
7820	* @param pVCpu The cross context virtual CPU structure.
7821	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7822	* be out-of-sync. Make sure to update the required
7823	* fields before using them.
7824	* @param u64IntInfo The VM-entry interruption-information field.
7825	* @param cbInstr The VM-entry instruction length in bytes (for
7826	* software interrupts, exceptions and privileged
7827	* software exceptions).
7828	* @param u32ErrCode The VM-entry exception error code.
7829	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7830	* @param puIntrState Pointer to the current guest interruptibility-state.
7831	* This interruptibility-state will be updated if
7832	* necessary. This cannot not be NULL.
7833	* @param fStepping Whether we're running in
7834	* hmR0VmxRunGuestCodeStep() and should return
7835	* VINF_EM_DBG_STEPPED if the event is injected
7836	* directly (register modified by us, not by
7837	* hardware on VM-entry).
7838	*
7839	* @remarks Requires CR0!
7840	* @remarks No-long-jump zone!!!
7841	*/
7842	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7843	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7844	uint32_t *puIntrState)
7845	{
7846	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7847	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7848	Assert(puIntrState);
7849	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7850
7851	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7852	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7853
7854	#ifdef VBOX_STRICT
7855	/* Validate the error-code-valid bit for hardware exceptions. */
7856	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7857	{
7858	switch (uVector)
7859	{
7860	case X86_XCPT_PF:
7861	case X86_XCPT_DF:
7862	case X86_XCPT_TS:
7863	case X86_XCPT_NP:
7864	case X86_XCPT_SS:
7865	case X86_XCPT_GP:
7866	case X86_XCPT_AC:
7867	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7868	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7869	/* fallthru */
7870	default:
7871	break;
7872	}
7873	}
7874	#endif
7875
7876	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7877	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7878	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7879
7880	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7881
7882	/* We require CR0 to check if the guest is in real-mode. */
7883	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7884	AssertRCReturn(rc, rc);
7885
7886	/*
7887	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7888	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7889	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7890	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7891	*/
7892	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7893	{
7894	PVM pVM = pVCpu->CTX_SUFF(pVM);
7895	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7896	{
7897	Assert(PDMVmmDevHeapIsEnabled(pVM));
7898	Assert(pVM->hm.s.vmx.pRealModeTSS);
7899
7900	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7901	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7902	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7903	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7904	AssertRCReturn(rc, rc);
7905	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7906
7907	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7908	size_t const cbIdtEntry = sizeof(X86IDTR16);
7909	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7910	{
7911	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7912	if (uVector == X86_XCPT_DF)
7913	return VINF_EM_RESET;
7914
7915	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7916	if (uVector == X86_XCPT_GP)
7917	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7918
7919	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7920	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7921	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
7922	fStepping, puIntrState);
7923	}
7924
7925	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7926	uint16_t uGuestIp = pMixedCtx->ip;
7927	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7928	{
7929	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7930	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7931	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7932	}
7933	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7934	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7935
7936	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7937	X86IDTR16 IdtEntry;
7938	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7939	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7940	AssertRCReturn(rc, rc);
7941
7942	/* Construct the stack frame for the interrupt/exception handler. */
7943	VBOXSTRICTRC rcStrict;
7944	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7945	if (rcStrict == VINF_SUCCESS)
7946	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7947	if (rcStrict == VINF_SUCCESS)
7948	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7949
7950	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7951	if (rcStrict == VINF_SUCCESS)
7952	{
7953	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7954	pMixedCtx->rip = IdtEntry.offSel;
7955	pMixedCtx->cs.Sel = IdtEntry.uSel;
7956	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7957	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7958	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7959	&& uVector == X86_XCPT_PF)
7960	pMixedCtx->cr2 = GCPtrFaultAddress;
7961
7962	/* If any other guest-state bits are changed here, make sure to update
7963	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7964	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7965	\| HM_CHANGED_GUEST_RIP
7966	\| HM_CHANGED_GUEST_RFLAGS
7967	\| HM_CHANGED_GUEST_RSP);
7968
7969	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7970	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7971	{
7972	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7973	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7974	Log4(("Clearing inhibition due to STI.\n"));
7975	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7976	}
7977	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7978	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7979
7980	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7981	it, if we are returning to ring-3 before executing guest code. */
7982	pVCpu->hm.s.Event.fPending = false;
7983
7984	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7985	if (fStepping)
7986	rcStrict = VINF_EM_DBG_STEPPED;
7987	}
7988	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7989	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7990	return rcStrict;
7991	}
7992
7993	/*
7994	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7995	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7996	*/
7997	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7998	}
7999
8000	/* Validate. */
8001	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8002	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8003	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8004
8005	/* Inject. */
8006	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8007	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8008	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8009	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8010
8011	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8012	&& uVector == X86_XCPT_PF)
8013	pMixedCtx->cr2 = GCPtrFaultAddress;
8014
8015	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8016	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8017
8018	AssertRCReturn(rc, rc);
8019	return VINF_SUCCESS;
8020	}
8021
8022
8023	/**
8024	* Clears the interrupt-window exiting control in the VMCS and if necessary
8025	* clears the current event in the VMCS as well.
8026	*
8027	* @returns VBox status code.
8028	* @param pVCpu The cross context virtual CPU structure.
8029	*
8030	* @remarks Use this function only to clear events that have not yet been
8031	* delivered to the guest but are injected in the VMCS!
8032	* @remarks No-long-jump zone!!!
8033	*/
8034	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8035	{
8036	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8037
8038	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8039	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8040
8041	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8042	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8043	}
8044
8045
8046	/**
8047	* Enters the VT-x session.
8048	*
8049	* @returns VBox status code.
8050	* @param pVM The cross context VM structure.
8051	* @param pVCpu The cross context virtual CPU structure.
8052	* @param pCpu Pointer to the CPU info struct.
8053	*/
8054	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8055	{
8056	AssertPtr(pVM);
8057	AssertPtr(pVCpu);
8058	Assert(pVM->hm.s.vmx.fSupported);
8059	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8060	NOREF(pCpu); NOREF(pVM);
8061
8062	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8063	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8064
8065	#ifdef VBOX_STRICT
8066	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8067	RTCCUINTREG uHostCR4 = ASMGetCR4();
8068	if (!(uHostCR4 & X86_CR4_VMXE))
8069	{
8070	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8071	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8072	}
8073	#endif
8074
8075	/*
8076	* Load the VCPU's VMCS as the current (and active) one.
8077	*/
8078	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8079	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8080	if (RT_FAILURE(rc))
8081	return rc;
8082
8083	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8084	pVCpu->hm.s.fLeaveDone = false;
8085	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8086
8087	return VINF_SUCCESS;
8088	}
8089
8090
8091	/**
8092	* The thread-context callback (only on platforms which support it).
8093	*
8094	* @param enmEvent The thread-context event.
8095	* @param pVCpu The cross context virtual CPU structure.
8096	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8097	* @thread EMT(pVCpu)
8098	*/
8099	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8100	{
8101	NOREF(fGlobalInit);
8102
8103	switch (enmEvent)
8104	{
8105	case RTTHREADCTXEVENT_OUT:
8106	{
8107	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8108	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8109	VMCPU_ASSERT_EMT(pVCpu);
8110
8111	PVM pVM = pVCpu->CTX_SUFF(pVM);
8112	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8113
8114	/* No longjmps (logger flushes, locks) in this fragile context. */
8115	VMMRZCallRing3Disable(pVCpu);
8116	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8117
8118	/*
8119	* Restore host-state (FPU, debug etc.)
8120	*/
8121	if (!pVCpu->hm.s.fLeaveDone)
8122	{
8123	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8124	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8125	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8126	pVCpu->hm.s.fLeaveDone = true;
8127	}
8128
8129	/* Leave HM context, takes care of local init (term). */
8130	int rc = HMR0LeaveCpu(pVCpu);
8131	AssertRC(rc); NOREF(rc);
8132
8133	/* Restore longjmp state. */
8134	VMMRZCallRing3Enable(pVCpu);
8135	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8136	break;
8137	}
8138
8139	case RTTHREADCTXEVENT_IN:
8140	{
8141	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8142	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8143	VMCPU_ASSERT_EMT(pVCpu);
8144
8145	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8146	VMMRZCallRing3Disable(pVCpu);
8147	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8148
8149	/* Initialize the bare minimum state required for HM. This takes care of
8150	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8151	int rc = HMR0EnterCpu(pVCpu);
8152	AssertRC(rc);
8153	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8154
8155	/* Load the active VMCS as the current one. */
8156	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8157	{
8158	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8159	AssertRC(rc); NOREF(rc);
8160	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8161	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8162	}
8163	pVCpu->hm.s.fLeaveDone = false;
8164
8165	/* Restore longjmp state. */
8166	VMMRZCallRing3Enable(pVCpu);
8167	break;
8168	}
8169
8170	default:
8171	break;
8172	}
8173	}
8174
8175
8176	/**
8177	* Saves the host state in the VMCS host-state.
8178	* Sets up the VM-exit MSR-load area.
8179	*
8180	* The CPU state will be loaded from these fields on every successful VM-exit.
8181	*
8182	* @returns VBox status code.
8183	* @param pVM The cross context VM structure.
8184	* @param pVCpu The cross context virtual CPU structure.
8185	*
8186	* @remarks No-long-jump zone!!!
8187	*/
8188	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8189	{
8190	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8191
8192	int rc = VINF_SUCCESS;
8193	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8194	{
8195	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8196	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8197
8198	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8199	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8200
8201	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8202	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8203
8204	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8205	}
8206	return rc;
8207	}
8208
8209
8210	/**
8211	* Saves the host state in the VMCS host-state.
8212	*
8213	* @returns VBox status code.
8214	* @param pVM The cross context VM structure.
8215	* @param pVCpu The cross context virtual CPU structure.
8216	*
8217	* @remarks No-long-jump zone!!!
8218	*/
8219	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8220	{
8221	AssertPtr(pVM);
8222	AssertPtr(pVCpu);
8223
8224	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8225
8226	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8227	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8228	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8229	return hmR0VmxSaveHostState(pVM, pVCpu);
8230	}
8231
8232
8233	/**
8234	* Loads the guest state into the VMCS guest-state area.
8235	*
8236	* The will typically be done before VM-entry when the guest-CPU state and the
8237	* VMCS state may potentially be out of sync.
8238	*
8239	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8240	* VM-entry controls.
8241	* Sets up the appropriate VMX non-root function to execute guest code based on
8242	* the guest CPU mode.
8243	*
8244	* @returns VBox strict status code.
8245	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8246	* without unrestricted guest access and the VMMDev is not presently
8247	* mapped (e.g. EFI32).
8248	*
8249	* @param pVM The cross context VM structure.
8250	* @param pVCpu The cross context virtual CPU structure.
8251	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8252	* out-of-sync. Make sure to update the required fields
8253	* before using them.
8254	*
8255	* @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8256	* caller disables then again on successfull return. Confusing.)
8257	*/
8258	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8259	{
8260	AssertPtr(pVM);
8261	AssertPtr(pVCpu);
8262	AssertPtr(pMixedCtx);
8263	HMVMX_ASSERT_PREEMPT_SAFE();
8264
8265	VMMRZCallRing3Disable(pVCpu);
8266	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8267
8268	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8269
8270	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8271
8272	/* Determine real-on-v86 mode. */
8273	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8274	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8275	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8276	{
8277	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8278	}
8279
8280	/*
8281	* Load the guest-state into the VMCS.
8282	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8283	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8284	*/
8285	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8286	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8289	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8290	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8291
8292	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8293	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8294	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8295
8296	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8297	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8298
8299	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8300	if (rcStrict == VINF_SUCCESS)
8301	{ /* likely */ }
8302	else
8303	{
8304	VMMRZCallRing3Enable(pVCpu);
8305	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8306	return rcStrict;
8307	}
8308
8309	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8310	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8311	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8312
8313	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8314	determine we don't have to swap EFER after all. */
8315	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8317
8318	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8320
8321	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8323
8324	/*
8325	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8326	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8327	*/
8328	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8329	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8330
8331	/* Clear any unused and reserved bits. */
8332	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8333
8334	VMMRZCallRing3Enable(pVCpu);
8335
8336	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8337	return rc;
8338	}
8339
8340
8341	/**
8342	* Loads the state shared between the host and guest into the VMCS.
8343	*
8344	* @param pVM The cross context VM structure.
8345	* @param pVCpu The cross context virtual CPU structure.
8346	* @param pCtx Pointer to the guest-CPU context.
8347	*
8348	* @remarks No-long-jump zone!!!
8349	*/
8350	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8351	{
8352	NOREF(pVM);
8353
8354	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8355	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8356
8357	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8358	{
8359	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8360	AssertRC(rc);
8361	}
8362
8363	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8364	{
8365	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8366	AssertRC(rc);
8367
8368	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8369	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8370	{
8371	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8372	AssertRC(rc);
8373	}
8374	}
8375
8376	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8377	{
8378	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8379	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8380	}
8381
8382	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8383	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8384	{
8385	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8386	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8387	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8388	AssertRC(rc);
8389	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8390	}
8391
8392	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8393	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8394	}
8395
8396
8397	/**
8398	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8399	*
8400	* @returns Strict VBox status code (i.e. informational status codes too).
8401	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8402	* without unrestricted guest access and the VMMDev is not presently
8403	* mapped (e.g. EFI32).
8404	*
8405	* @param pVM The cross context VM structure.
8406	* @param pVCpu The cross context virtual CPU structure.
8407	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8408	* out-of-sync. Make sure to update the required fields
8409	* before using them.
8410	*/
8411	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8412	{
8413	HMVMX_ASSERT_PREEMPT_SAFE();
8414
8415	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8416	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8417	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8418	#endif
8419
8420	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8421	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8422	{
8423	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8424	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8425	{ /* likely */}
8426	else
8427	{
8428	AssertLogRelMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8429	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8430	}
8431	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8432	}
8433	else if (HMCPU_CF_VALUE(pVCpu))
8434	{
8435	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8436	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8437	{ /* likely */}
8438	else
8439	{
8440	AssertLogRelMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8441	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8442	return rcStrict;
8443	}
8444	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8445	}
8446
8447	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8448	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8449	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8450	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8451	return rcStrict;
8452	}
8453
8454
8455	/**
8456	* Does the preparations before executing guest code in VT-x.
8457	*
8458	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8459	* recompiler/IEM. We must be cautious what we do here regarding committing
8460	* guest-state information into the VMCS assuming we assuredly execute the
8461	* guest in VT-x mode.
8462	*
8463	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8464	* the common-state (TRPM/forceflags), we must undo those changes so that the
8465	* recompiler/IEM can (and should) use them when it resumes guest execution.
8466	* Otherwise such operations must be done when we can no longer exit to ring-3.
8467	*
8468	* @returns Strict VBox status code (i.e. informational status codes too).
8469	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8470	* have been disabled.
8471	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8472	* double-fault into the guest.
8473	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8474	* dispatched directly.
8475	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8476	*
8477	* @param pVM The cross context VM structure.
8478	* @param pVCpu The cross context virtual CPU structure.
8479	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8480	* out-of-sync. Make sure to update the required fields
8481	* before using them.
8482	* @param pVmxTransient Pointer to the VMX transient structure.
8483	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8484	* us ignore some of the reasons for returning to
8485	* ring-3, and return VINF_EM_DBG_STEPPED if event
8486	* dispatching took place.
8487	*/
8488	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8489	{
8490	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8491
8492	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8493	PGMRZDynMapFlushAutoSet(pVCpu);
8494	#endif
8495
8496	/* Check force flag actions that might require us to go back to ring-3. */
8497	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8498	if (rcStrict == VINF_SUCCESS)
8499	{ /* FFs doesn't get set all the time. */ }
8500	else
8501	return rcStrict;
8502
8503	#ifndef IEM_VERIFICATION_MODE_FULL
8504	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8505	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8506	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8507	{
8508	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8509	RTGCPHYS GCPhysApicBase;
8510	GCPhysApicBase = pMixedCtx->msrApicBase;
8511	GCPhysApicBase &= PAGE_BASE_GC_MASK;
8512
8513	/* Unalias any existing mapping. */
8514	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8515	AssertRCReturn(rc, rc);
8516
8517	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8518	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8519	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8520	AssertRCReturn(rc, rc);
8521
8522	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8523	}
8524	#endif /* !IEM_VERIFICATION_MODE_FULL */
8525
8526	if (TRPMHasTrap(pVCpu))
8527	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8528	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8529
8530	/*
8531	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8532	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8533	*/
8534	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8535	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8536	{ /* likely */ }
8537	else
8538	{
8539	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8540	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8541	return rcStrict;
8542	}
8543
8544	/*
8545	* Load the guest state bits, we can handle longjmps/getting preempted here.
8546	*
8547	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8548	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8549	* Hence, this needs to be done -after- injection of events.
8550	*/
8551	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8552	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8553	{ /* likely */ }
8554	else
8555	return rcStrict;
8556
8557	/*
8558	* No longjmps to ring-3 from this point on!!!
8559	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8560	* This also disables flushing of the R0-logger instance (if any).
8561	*/
8562	VMMRZCallRing3Disable(pVCpu);
8563
8564	/*
8565	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8566	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8567	*
8568	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8569	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8570	*
8571	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8572	* executing guest code.
8573	*/
8574	pVmxTransient->fEFlags = ASMIntDisableFlags();
8575
8576	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8577	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8578	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8579	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8580	{
8581	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8582	{
8583	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8584	pVCpu->hm.s.Event.fPending = false;
8585
8586	return VINF_SUCCESS;
8587	}
8588
8589	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8590	rcStrict = VINF_EM_RAW_INTERRUPT;
8591	}
8592	else
8593	{
8594	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8595	rcStrict = VINF_EM_RAW_TO_R3;
8596	}
8597
8598	ASMSetFlags(pVmxTransient->fEFlags);
8599	VMMRZCallRing3Enable(pVCpu);
8600
8601	return rcStrict;
8602	}
8603
8604
8605	/**
8606	* Prepares to run guest code in VT-x and we've committed to doing so. This
8607	* means there is no backing out to ring-3 or anywhere else at this
8608	* point.
8609	*
8610	* @param pVM The cross context VM structure.
8611	* @param pVCpu The cross context virtual CPU structure.
8612	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8613	* out-of-sync. Make sure to update the required fields
8614	* before using them.
8615	* @param pVmxTransient Pointer to the VMX transient structure.
8616	*
8617	* @remarks Called with preemption disabled.
8618	* @remarks No-long-jump zone!!!
8619	*/
8620	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8621	{
8622	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8623	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8624	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8625
8626	/*
8627	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8628	*/
8629	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8630	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8631
8632	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8633	if (!CPUMIsGuestFPUStateActive(pVCpu))
8634	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8635	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8636	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8637	#endif
8638
8639	if ( pVCpu->hm.s.fPreloadGuestFpu
8640	&& !CPUMIsGuestFPUStateActive(pVCpu))
8641	{
8642	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8643	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8644	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8645	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8646	}
8647
8648	/*
8649	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8650	*/
8651	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8652	&& pVCpu->hm.s.vmx.cMsrs > 0)
8653	{
8654	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8655	}
8656
8657	/*
8658	* Load the host state bits as we may've been preempted (only happens when
8659	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8660	*/
8661	/** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8662	* any effect to the host state needing to be saved? */
8663	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8664	{
8665	/* This ASSUMES that pfnStartVM has been set up already. */
8666	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8667	AssertRC(rc);
8668	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8669	}
8670	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8671
8672	/*
8673	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8674	*/
8675	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8676	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8677	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8678
8679	/* Store status of the shared guest-host state at the time of VM-entry. */
8680	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8681	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8682	{
8683	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8684	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8685	}
8686	else
8687	#endif
8688	{
8689	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8690	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8691	}
8692	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8693
8694	/*
8695	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8696	*/
8697	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8698	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8699
8700	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8701	RTCPUID idCurrentCpu = pCpu->idCpu;
8702	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8703	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8704	{
8705	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8706	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8707	}
8708
8709	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8710	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8711	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8712	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8713
8714	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8715
8716	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8717	to start executing. */
8718
8719	/*
8720	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8721	*/
8722	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8723	{
8724	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8725	{
8726	bool fMsrUpdated;
8727	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8728	AssertRC(rc2);
8729	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8730
8731	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8732	&fMsrUpdated);
8733	AssertRC(rc2);
8734	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8735
8736	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8737	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8738	}
8739	else
8740	{
8741	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8742	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8743	}
8744	}
8745
8746	#ifdef VBOX_STRICT
8747	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8748	hmR0VmxCheckHostEferMsr(pVCpu);
8749	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8750	#endif
8751	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8752	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8753	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8754	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8755	#endif
8756	}
8757
8758
8759	/**
8760	* Performs some essential restoration of state after running guest code in
8761	* VT-x.
8762	*
8763	* @param pVM The cross context VM structure.
8764	* @param pVCpu The cross context virtual CPU structure.
8765	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8766	* out-of-sync. Make sure to update the required fields
8767	* before using them.
8768	* @param pVmxTransient Pointer to the VMX transient structure.
8769	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8770	*
8771	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
8772	*
8773	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8774	* unconditionally when it is safe to do so.
8775	*/
8776	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8777	{
8778	NOREF(pVM);
8779
8780	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8781
8782	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8783	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8784	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8785	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8786	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8787	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8788
8789	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8790	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8791
8792	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8793	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8794	Assert(!ASMIntAreEnabled());
8795	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8796
8797	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8798	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8799	{
8800	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8801	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8802	}
8803	#endif
8804
8805	#if HC_ARCH_BITS == 64
8806	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8807	#endif
8808	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8809	#ifdef VBOX_STRICT
8810	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8811	#endif
8812	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8813	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8814
8815	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8816	uint32_t uExitReason;
8817	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8818	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8819	AssertRC(rc);
8820	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8821	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8822
8823	/* Update the VM-exit history array. */
8824	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8825
8826	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8827	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8828	{
8829	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8830	pVmxTransient->fVMEntryFailed));
8831	return;
8832	}
8833
8834	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8835	{
8836	/** @todo We can optimize this by only syncing with our force-flags when
8837	* really needed and keeping the VMCS state as it is for most
8838	* VM-exits. */
8839	/* Update the guest interruptibility-state from the VMCS. */
8840	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8841
8842	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8843	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8844	AssertRC(rc);
8845	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8846	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8847	AssertRC(rc);
8848	#endif
8849
8850	/*
8851	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8852	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8853	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8854	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8855	*/
8856	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8857	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8858	{
8859	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8860	AssertRC(rc);
8861	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8862	}
8863	}
8864	}
8865
8866
8867	/**
8868	* Runs the guest code using VT-x the normal way.
8869	*
8870	* @returns VBox status code.
8871	* @param pVM The cross context VM structure.
8872	* @param pVCpu The cross context virtual CPU structure.
8873	* @param pCtx Pointer to the guest-CPU context.
8874	*
8875	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8876	*/
8877	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8878	{
8879	VMXTRANSIENT VmxTransient;
8880	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8881	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8882	uint32_t cLoops = 0;
8883
8884	for (;; cLoops++)
8885	{
8886	Assert(!HMR0SuspendPending());
8887	HMVMX_ASSERT_CPU_SAFE();
8888
8889	/* Preparatory work for running guest code, this may force us to return
8890	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8891	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8892	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8893	if (rcStrict != VINF_SUCCESS)
8894	break;
8895
8896	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8897	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8898	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8899
8900	/* Restore any residual host-state and save any bits shared between host
8901	and guest into the guest-CPU state. Re-enables interrupts! */
8902	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8903
8904	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8905	if (RT_SUCCESS(rcRun))
8906	{ /* very likely */ }
8907	else
8908	{
8909	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8910	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8911	return rcRun;
8912	}
8913
8914	/* Profile the VM-exit. */
8915	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8917	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8918	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8919	HMVMX_START_EXIT_DISPATCH_PROF();
8920
8921	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8922
8923	/* Handle the VM-exit. */
8924	#ifdef HMVMX_USE_FUNCTION_TABLE
8925	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8926	#else
8927	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8928	#endif
8929	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8930	if (rcStrict == VINF_SUCCESS)
8931	{
8932	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8933	continue; /* likely */
8934	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8935	rcStrict = VINF_EM_RAW_INTERRUPT;
8936	}
8937	break;
8938	}
8939
8940	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8941	return rcStrict;
8942	}
8943
8944
8945
8946	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8947	* probes.
8948	*
8949	* The following few functions and associated structure contains the bloat
8950	* necessary for providing detailed debug events and dtrace probes as well as
8951	* reliable host side single stepping. This works on the principle of
8952	* "subclassing" the normal execution loop and workers. We replace the loop
8953	* method completely and override selected helpers to add necessary adjustments
8954	* to their core operation.
8955	*
8956	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8957	* any performance for debug and analysis features.
8958	*
8959	* @{
8960	*/
8961
8962	typedef struct VMXRUNDBGSTATE
8963	{
8964	/** The RIP we started executing at. This is for detecting that we stepped. */
8965	uint64_t uRipStart;
8966	/** The CS we started executing with. */
8967	uint16_t uCsStart;
8968
8969	/** Whether we've actually modified the 1st execution control field. */
8970	bool fModifiedProcCtls : 1;
8971	/** Whether we've actually modified the 2nd execution control field. */
8972	bool fModifiedProcCtls2 : 1;
8973	/** Whether we've actually modified the exception bitmap. */
8974	bool fModifiedXcptBitmap : 1;
8975
8976	/** We desire the modified the CR0 mask to be cleared. */
8977	bool fClearCr0Mask : 1;
8978	/** We desire the modified the CR4 mask to be cleared. */
8979	bool fClearCr4Mask : 1;
8980	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8981	uint32_t fCpe1Extra;
8982	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8983	uint32_t fCpe1Unwanted;
8984	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8985	uint32_t fCpe2Extra;
8986	/** Extra stuff we need in */
8987	uint32_t bmXcptExtra;
8988	/** The sequence number of the Dtrace provider settings the state was
8989	* configured against. */
8990	uint32_t uDtraceSettingsSeqNo;
8991	/** Exits to check (one bit per exit). */
8992	uint32_t bmExitsToCheck[3];
8993
8994	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8995	uint32_t fProcCtlsInitial;
8996	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8997	uint32_t fProcCtls2Initial;
8998	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8999	uint32_t bmXcptInitial;
9000	} VMXRUNDBGSTATE;
9001	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9002	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9003
9004
9005	/**
9006	* Initializes the VMXRUNDBGSTATE structure.
9007	*
9008	* @param pVCpu The cross context virtual CPU structure of the
9009	* calling EMT.
9010	* @param pCtx The CPU register context to go with @a pVCpu.
9011	* @param pDbgState The structure to initialize.
9012	*/
9013	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9014	{
9015	pDbgState->uRipStart = pCtx->rip;
9016	pDbgState->uCsStart = pCtx->cs.Sel;
9017
9018	pDbgState->fModifiedProcCtls = false;
9019	pDbgState->fModifiedProcCtls2 = false;
9020	pDbgState->fModifiedXcptBitmap = false;
9021	pDbgState->fClearCr0Mask = false;
9022	pDbgState->fClearCr4Mask = false;
9023	pDbgState->fCpe1Extra = 0;
9024	pDbgState->fCpe1Unwanted = 0;
9025	pDbgState->fCpe2Extra = 0;
9026	pDbgState->bmXcptExtra = 0;
9027	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9028	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9029	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9030	}
9031
9032
9033	/**
9034	* Updates the VMSC fields with changes requested by @a pDbgState.
9035	*
9036	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9037	* immediately before executing guest code, i.e. when interrupts are disabled.
9038	* We don't check status codes here as we cannot easily assert or return in the
9039	* latter case.
9040	*
9041	* @param pVCpu The cross context virtual CPU structure.
9042	* @param pDbgState The debug state.
9043	*/
9044	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9045	{
9046	/*
9047	* Ensure desired flags in VMCS control fields are set.
9048	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9049	*
9050	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9051	* there should be no stale data in pCtx at this point.
9052	*/
9053	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9054	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9055	{
9056	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9057	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9058	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9059	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9060	pDbgState->fModifiedProcCtls = true;
9061	}
9062
9063	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9064	{
9065	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9066	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9067	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9068	pDbgState->fModifiedProcCtls2 = true;
9069	}
9070
9071	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9072	{
9073	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9074	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9075	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9076	pDbgState->fModifiedXcptBitmap = true;
9077	}
9078
9079	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9080	{
9081	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9082	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9083	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9084	}
9085
9086	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9087	{
9088	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9089	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9090	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9091	}
9092	}
9093
9094
9095	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9096	{
9097	/*
9098	* Restore exit control settings as we may not reenter this function the
9099	* next time around.
9100	*/
9101	/* We reload the initial value, trigger what we can of recalculations the
9102	next time around. From the looks of things, that's all that's required atm. */
9103	if (pDbgState->fModifiedProcCtls)
9104	{
9105	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9106	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9107	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9108	AssertRCReturn(rc2, rc2);
9109	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9110	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_DEBUG);
9111	}
9112
9113	/* We're currently the only ones messing with this one, so just restore the
9114	cached value and reload the field. */
9115	if ( pDbgState->fModifiedProcCtls2
9116	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9117	{
9118	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9119	AssertRCReturn(rc2, rc2);
9120	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9121	}
9122
9123	/* If we've modified the exception bitmap, we restore it and trigger
9124	reloading and partial recalculation the next time around. */
9125	if (pDbgState->fModifiedXcptBitmap)
9126	{
9127	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9128	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS \| HM_CHANGED_GUEST_CR0);
9129	}
9130
9131	/* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9132	if (pDbgState->fClearCr0Mask)
9133	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9134
9135	/* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9136	if (pDbgState->fClearCr4Mask)
9137	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9138
9139	return rcStrict;
9140	}
9141
9142
9143	/**
9144	* Configures VM-exit controls for current DBGF and DTrace settings.
9145	*
9146	* This updates @a pDbgState and the VMCS execution control fields to reflect
9147	* the necessary exits demanded by DBGF and DTrace.
9148	*
9149	* @param pVM The cross context VM structure.
9150	* @param pVCpu The cross context virtual CPU structure.
9151	* @param pCtx Pointer to the guest-CPU context.
9152	* @param pDbgState The debug state.
9153	* @param pVmxTransient Pointer to the VMX transient structure. May update
9154	* fUpdateTscOffsettingAndPreemptTimer.
9155	*/
9156	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9157	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9158	{
9159	/*
9160	* Take down the dtrace serial number so we can spot changes.
9161	*/
9162	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9163	ASMCompilerBarrier();
9164
9165	/*
9166	* We'll rebuild most of the middle block of data members (holding the
9167	* current settings) as we go along here, so start by clearing it all.
9168	*/
9169	pDbgState->bmXcptExtra = 0;
9170	pDbgState->fCpe1Extra = 0;
9171	pDbgState->fCpe1Unwanted = 0;
9172	pDbgState->fCpe2Extra = 0;
9173	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9174	pDbgState->bmExitsToCheck[i] = 0;
9175
9176	/*
9177	* Software interrupts (INT XXh) - no idea how to trigger these...
9178	*/
9179	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9180	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9181	{
9182	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9183	}
9184
9185	/*
9186	* Exception bitmap and XCPT events+probes.
9187	*/
9188	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9189	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9190	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9191
9192	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9193	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9194	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9195	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9196	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9197	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9198	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9199	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9200	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9201	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9202	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9203	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9204	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9205	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9206	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9207	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9208	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9209	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9210
9211	if (pDbgState->bmXcptExtra)
9212	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9213
9214	/*
9215	* Process events and probes for VM exits, making sure we get the wanted exits.
9216	*
9217	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9218	* So, when adding/changing/removing please don't forget to update it.
9219	*
9220	* Some of the macros are picking up local variables to save horizontal space,
9221	* (being able to see it in a table is the lesser evil here).
9222	*/
9223	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9224	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9225	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9226	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9227	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9228	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9229	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9230	} else do { } while (0)
9231	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9232	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9233	{ \
9234	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9235	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9236	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9237	} else do { } while (0)
9238	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9239	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9240	{ \
9241	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9242	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9243	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9244	} else do { } while (0)
9245	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9246	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9247	{ \
9248	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9249	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9250	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9251	} else do { } while (0)
9252
9253	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9254	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9255	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9256	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9257	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9258
9259	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9260	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9261	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9262	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9263	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9264	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9265	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9266	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9267	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9268	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9269	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9270	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9271	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9272	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9273	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9274	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9275	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9276	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9277	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9278	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9279	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9280	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9281	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9282	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9283	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9284	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9285	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9286	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9287	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9288	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9289	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9290	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9291	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9292	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9293	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9294	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9295
9296	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9297	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9298	{
9299	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9300	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9301	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9302	AssertRC(rc2);
9303
9304	#if 0 /** @todo fix me */
9305	pDbgState->fClearCr0Mask = true;
9306	pDbgState->fClearCr4Mask = true;
9307	#endif
9308	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9309	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9310	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9311	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9312	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9313	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9314	require clearing here and in the loop if we start using it. */
9315	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9316	}
9317	else
9318	{
9319	if (pDbgState->fClearCr0Mask)
9320	{
9321	pDbgState->fClearCr0Mask = false;
9322	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9323	}
9324	if (pDbgState->fClearCr4Mask)
9325	{
9326	pDbgState->fClearCr4Mask = false;
9327	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9328	}
9329	}
9330	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9331	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9332
9333	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9334	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9335	{
9336	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9337	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9338	}
9339	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9340	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9341
9342	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9343	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9344	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9345	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9346	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9347	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9348	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9349	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9350	#if 0 /** @todo too slow, fix handler. */
9351	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9352	#endif
9353	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9354
9355	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9356	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9357	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9358	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9359	{
9360	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9361	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9362	}
9363	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9364	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9365	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9366	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9367
9368	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9369	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9370	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9371	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9372	{
9373	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9374	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9375	}
9376	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9377	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9378	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9379	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9380
9381	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9382	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9383	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9384	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9385	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9386	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9387	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9388	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9389	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9390	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9391	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9392	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9393	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9394	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9395	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9396	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9397	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9398	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9399	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9400	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9401	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9402	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9403
9404	#undef IS_EITHER_ENABLED
9405	#undef SET_ONLY_XBM_IF_EITHER_EN
9406	#undef SET_CPE1_XBM_IF_EITHER_EN
9407	#undef SET_CPEU_XBM_IF_EITHER_EN
9408	#undef SET_CPE2_XBM_IF_EITHER_EN
9409
9410	/*
9411	* Sanitize the control stuff.
9412	*/
9413	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9414	if (pDbgState->fCpe2Extra)
9415	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9416	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9417	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9418	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9419	{
9420	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9421	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9422	}
9423
9424	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9425	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9426	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9427	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9428	}
9429
9430
9431	/**
9432	* Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9433	*
9434	* The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9435	* bitmap. The caller has checked for NMIs already, so we don't have to do that
9436	* either.
9437	*
9438	* @returns Strict VBox status code (i.e. informational status codes too).
9439	* @param pVM The cross context VM structure.
9440	* @param pVCpu The cross context virtual CPU structure.
9441	* @param pMixedCtx Pointer to the guest-CPU context.
9442	* @param pVmxTransient Pointer to the VMX-transient structure.
9443	* @param uExitReason The VM-exit reason.
9444	*
9445	* @remarks The name of this function is displayed by dtrace, so keep it short
9446	* and to the point. No longer than 33 chars long, please.
9447	*/
9448	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9449	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9450	{
9451	/*
9452	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9453	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9454	*
9455	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9456	* does. Must add/change/remove both places. Same ordering, please.
9457	*
9458	* Added/removed events must also be reflected in the next section
9459	* where we dispatch dtrace events.
9460	*/
9461	bool fDtrace1 = false;
9462	bool fDtrace2 = false;
9463	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9464	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9465	uint32_t uEventArg = 0;
9466	#define SET_EXIT(a_EventSubName) \
9467	do { \
9468	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9469	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9470	} while (0)
9471	#define SET_BOTH(a_EventSubName) \
9472	do { \
9473	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9474	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9475	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9476	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9477	} while (0)
9478	switch (uExitReason)
9479	{
9480	case VMX_EXIT_MTF:
9481	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9482
9483	case VMX_EXIT_XCPT_OR_NMI:
9484	{
9485	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9486	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9487	{
9488	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9489	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9490	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9491	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9492	{
9493	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9494	{
9495	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9496	uEventArg = pVmxTransient->uExitIntErrorCode;
9497	}
9498	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9499	switch (enmEvent1)
9500	{
9501	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9502	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9503	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9504	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9505	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9506	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9507	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9508	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9509	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9510	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9511	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9512	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9513	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9514	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9515	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9516	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9517	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9518	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9519	default: break;
9520	}
9521	}
9522	else
9523	AssertFailed();
9524	break;
9525
9526	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9527	uEventArg = idxVector;
9528	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9529	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9530	break;
9531	}
9532	break;
9533	}
9534
9535	case VMX_EXIT_TRIPLE_FAULT:
9536	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9537	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9538	break;
9539	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9540	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9541	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9542	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9543	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9544
9545	/* Instruction specific VM-exits: */
9546	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9547	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9548	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9549	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9550	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9551	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9552	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9553	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9554	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9555	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9556	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9557	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9558	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9559	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9560	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9561	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9562	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9563	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9564	case VMX_EXIT_MOV_CRX:
9565	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9566	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9567	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9568	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9569	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9570	SET_BOTH(CRX_READ);
9571	else
9572	SET_BOTH(CRX_WRITE);
9573	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9574	break;
9575	case VMX_EXIT_MOV_DRX:
9576	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9577	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9578	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9579	SET_BOTH(DRX_READ);
9580	else
9581	SET_BOTH(DRX_WRITE);
9582	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9583	break;
9584	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9585	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9586	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9587	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9588	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9589	case VMX_EXIT_XDTR_ACCESS:
9590	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9591	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9592	{
9593	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9594	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9595	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9596	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9597	}
9598	break;
9599
9600	case VMX_EXIT_TR_ACCESS:
9601	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9602	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9603	{
9604	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9605	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9606	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9607	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9608	}
9609	break;
9610
9611	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9612	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9613	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9614	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9615	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9616	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9617	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9618	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9619	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9620	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9621	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9622
9623	/* Events that aren't relevant at this point. */
9624	case VMX_EXIT_EXT_INT:
9625	case VMX_EXIT_INT_WINDOW:
9626	case VMX_EXIT_NMI_WINDOW:
9627	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9628	case VMX_EXIT_PREEMPT_TIMER:
9629	case VMX_EXIT_IO_INSTR:
9630	break;
9631
9632	/* Errors and unexpected events. */
9633	case VMX_EXIT_INIT_SIGNAL:
9634	case VMX_EXIT_SIPI:
9635	case VMX_EXIT_IO_SMI:
9636	case VMX_EXIT_SMI:
9637	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9638	case VMX_EXIT_ERR_MSR_LOAD:
9639	case VMX_EXIT_ERR_MACHINE_CHECK:
9640	break;
9641
9642	default:
9643	AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9644	break;
9645	}
9646	#undef SET_BOTH
9647	#undef SET_EXIT
9648
9649	/*
9650	* Dtrace tracepoints go first. We do them here at once so we don't
9651	* have to copy the guest state saving and stuff a few dozen times.
9652	* Down side is that we've got to repeat the switch, though this time
9653	* we use enmEvent since the probes are a subset of what DBGF does.
9654	*/
9655	if (fDtrace1 \|\| fDtrace2)
9656	{
9657	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9658	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9659	switch (enmEvent1)
9660	{
9661	/** @todo consider which extra parameters would be helpful for each probe. */
9662	case DBGFEVENT_END: break;
9663	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9664	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9665	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9666	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9667	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9668	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9669	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9670	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9671	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9672	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9673	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9674	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9675	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9676	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9677	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9678	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9679	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9680	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9681	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9682	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9683	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9684	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9685	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9686	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9687	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9688	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9689	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9690	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9691	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9692	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9693	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9694	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9695	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9696	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9697	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9698	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9699	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9700	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9701	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9702	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9703	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9704	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9705	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9706	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9707	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9708	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9709	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9710	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9711	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9712	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9713	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9714	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9715	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9716	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9717	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9718	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9719	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9720	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9721	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9722	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9723	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9724	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9725	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9726	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9727	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9728	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9729	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9730	}
9731	switch (enmEvent2)
9732	{
9733	/** @todo consider which extra parameters would be helpful for each probe. */
9734	case DBGFEVENT_END: break;
9735	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9736	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9737	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9738	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9739	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9740	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9741	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9742	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9743	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9744	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9745	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9746	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9747	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9748	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9749	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9750	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9751	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9752	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9753	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9754	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9755	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9756	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9757	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9758	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9759	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9760	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9761	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9762	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9763	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9764	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9765	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9766	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9767	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9768	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9769	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9770	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9771	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9772	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9773	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9774	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9775	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9776	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9777	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9778	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9779	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9780	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9781	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9782	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9783	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9784	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9785	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9786	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9787	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9788	}
9789	}
9790
9791	/*
9792	* Fire of the DBGF event, if enabled (our check here is just a quick one,
9793	* the DBGF call will do a full check).
9794	*
9795	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9796	* Note! If we have to events, we prioritize the first, i.e. the instruction
9797	* one, in order to avoid event nesting.
9798	*/
9799	if ( enmEvent1 != DBGFEVENT_END
9800	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9801	{
9802	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9803	if (rcStrict != VINF_SUCCESS)
9804	return rcStrict;
9805	}
9806	else if ( enmEvent2 != DBGFEVENT_END
9807	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9808	{
9809	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9810	if (rcStrict != VINF_SUCCESS)
9811	return rcStrict;
9812	}
9813
9814	return VINF_SUCCESS;
9815	}
9816
9817
9818	/**
9819	* Single-stepping VM-exit filtering.
9820	*
9821	* This is preprocessing the exits and deciding whether we've gotten far enough
9822	* to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9823	* performed.
9824	*
9825	* @returns Strict VBox status code (i.e. informational status codes too).
9826	* @param pVM The cross context VM structure.
9827	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9828	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9829	* out-of-sync. Make sure to update the required
9830	* fields before using them.
9831	* @param pVmxTransient Pointer to the VMX-transient structure.
9832	* @param uExitReason The VM-exit reason.
9833	* @param pDbgState The debug state.
9834	*/
9835	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9836	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9837	{
9838	/*
9839	* Expensive (saves context) generic dtrace exit probe.
9840	*/
9841	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9842	{ /* more likely */ }
9843	else
9844	{
9845	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9846	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9847	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9848	}
9849
9850	/*
9851	* Check for host NMI, just to get that out of the way.
9852	*/
9853	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9854	{ /* normally likely */ }
9855	else
9856	{
9857	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9858	AssertRCReturn(rc2, rc2);
9859	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9860	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9861	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9862	}
9863
9864	/*
9865	* Check for single stepping event if we're stepping.
9866	*/
9867	if (pVCpu->hm.s.fSingleInstruction)
9868	{
9869	switch (uExitReason)
9870	{
9871	case VMX_EXIT_MTF:
9872	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9873
9874	/* Various events: */
9875	case VMX_EXIT_XCPT_OR_NMI:
9876	case VMX_EXIT_EXT_INT:
9877	case VMX_EXIT_TRIPLE_FAULT:
9878	case VMX_EXIT_INT_WINDOW:
9879	case VMX_EXIT_NMI_WINDOW:
9880	case VMX_EXIT_TASK_SWITCH:
9881	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9882	case VMX_EXIT_APIC_ACCESS:
9883	case VMX_EXIT_EPT_VIOLATION:
9884	case VMX_EXIT_EPT_MISCONFIG:
9885	case VMX_EXIT_PREEMPT_TIMER:
9886
9887	/* Instruction specific VM-exits: */
9888	case VMX_EXIT_CPUID:
9889	case VMX_EXIT_GETSEC:
9890	case VMX_EXIT_HLT:
9891	case VMX_EXIT_INVD:
9892	case VMX_EXIT_INVLPG:
9893	case VMX_EXIT_RDPMC:
9894	case VMX_EXIT_RDTSC:
9895	case VMX_EXIT_RSM:
9896	case VMX_EXIT_VMCALL:
9897	case VMX_EXIT_VMCLEAR:
9898	case VMX_EXIT_VMLAUNCH:
9899	case VMX_EXIT_VMPTRLD:
9900	case VMX_EXIT_VMPTRST:
9901	case VMX_EXIT_VMREAD:
9902	case VMX_EXIT_VMRESUME:
9903	case VMX_EXIT_VMWRITE:
9904	case VMX_EXIT_VMXOFF:
9905	case VMX_EXIT_VMXON:
9906	case VMX_EXIT_MOV_CRX:
9907	case VMX_EXIT_MOV_DRX:
9908	case VMX_EXIT_IO_INSTR:
9909	case VMX_EXIT_RDMSR:
9910	case VMX_EXIT_WRMSR:
9911	case VMX_EXIT_MWAIT:
9912	case VMX_EXIT_MONITOR:
9913	case VMX_EXIT_PAUSE:
9914	case VMX_EXIT_XDTR_ACCESS:
9915	case VMX_EXIT_TR_ACCESS:
9916	case VMX_EXIT_INVEPT:
9917	case VMX_EXIT_RDTSCP:
9918	case VMX_EXIT_INVVPID:
9919	case VMX_EXIT_WBINVD:
9920	case VMX_EXIT_XSETBV:
9921	case VMX_EXIT_RDRAND:
9922	case VMX_EXIT_INVPCID:
9923	case VMX_EXIT_VMFUNC:
9924	case VMX_EXIT_RDSEED:
9925	case VMX_EXIT_XSAVES:
9926	case VMX_EXIT_XRSTORS:
9927	{
9928	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9929	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9930	AssertRCReturn(rc2, rc2);
9931	if ( pMixedCtx->rip != pDbgState->uRipStart
9932	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
9933	return VINF_EM_DBG_STEPPED;
9934	break;
9935	}
9936
9937	/* Errors and unexpected events: */
9938	case VMX_EXIT_INIT_SIGNAL:
9939	case VMX_EXIT_SIPI:
9940	case VMX_EXIT_IO_SMI:
9941	case VMX_EXIT_SMI:
9942	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9943	case VMX_EXIT_ERR_MSR_LOAD:
9944	case VMX_EXIT_ERR_MACHINE_CHECK:
9945	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9946	break;
9947
9948	default:
9949	AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9950	break;
9951	}
9952	}
9953
9954	/*
9955	* Check for debugger event breakpoints and dtrace probes.
9956	*/
9957	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9958	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9959	{
9960	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9961	if (rcStrict != VINF_SUCCESS)
9962	return rcStrict;
9963	}
9964
9965	/*
9966	* Normal processing.
9967	*/
9968	#ifdef HMVMX_USE_FUNCTION_TABLE
9969	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9970	#else
9971	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9972	#endif
9973	}
9974
9975
9976	/**
9977	* Single steps guest code using VT-x.
9978	*
9979	* @returns Strict VBox status code (i.e. informational status codes too).
9980	* @param pVM The cross context VM structure.
9981	* @param pVCpu The cross context virtual CPU structure.
9982	* @param pCtx Pointer to the guest-CPU context.
9983	*
9984	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9985	*/
9986	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9987	{
9988	VMXTRANSIENT VmxTransient;
9989	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9990
9991	/* Set HMCPU indicators. */
9992	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9993	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
9994	pVCpu->hm.s.fDebugWantRdTscExit = false;
9995	pVCpu->hm.s.fUsingDebugLoop = true;
9996
9997	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9998	VMXRUNDBGSTATE DbgState;
9999	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10000	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10001
10002	/*
10003	* The loop.
10004	*/
10005	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10006	for (uint32_t cLoops = 0; ; cLoops++)
10007	{
10008	Assert(!HMR0SuspendPending());
10009	HMVMX_ASSERT_CPU_SAFE();
10010	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10011
10012	/*
10013	* Preparatory work for running guest code, this may force us to return
10014	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10015	*/
10016	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10017	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10018	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10019	if (rcStrict != VINF_SUCCESS)
10020	break;
10021
10022	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10023	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10024
10025	/*
10026	* Now we can run the guest code.
10027	*/
10028	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10029
10030	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10031
10032	/*
10033	* Restore any residual host-state and save any bits shared between host
10034	* and guest into the guest-CPU state. Re-enables interrupts!
10035	*/
10036	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10037
10038	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10039	if (RT_SUCCESS(rcRun))
10040	{ /* very likely */ }
10041	else
10042	{
10043	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10044	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10045	return rcRun;
10046	}
10047
10048	/* Profile the VM-exit. */
10049	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10050	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10051	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10052	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10053	HMVMX_START_EXIT_DISPATCH_PROF();
10054
10055	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10056
10057	/*
10058	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10059	*/
10060	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10061	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10062	if (rcStrict != VINF_SUCCESS)
10063	break;
10064	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10065	{
10066	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10067	rcStrict = VINF_EM_RAW_INTERRUPT;
10068	break;
10069	}
10070
10071	/*
10072	* Stepping: Did the RIP change, if so, consider it a single step.
10073	* Otherwise, make sure one of the TFs gets set.
10074	*/
10075	if (fStepping)
10076	{
10077	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10078	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10079	AssertRCReturn(rc2, rc2);
10080	if ( pCtx->rip != DbgState.uRipStart
10081	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10082	{
10083	rcStrict = VINF_EM_DBG_STEPPED;
10084	break;
10085	}
10086	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10087	}
10088
10089	/*
10090	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10091	*/
10092	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10093	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10094	}
10095
10096	/*
10097	* Clear the X86_EFL_TF if necessary.
10098	*/
10099	if (pVCpu->hm.s.fClearTrapFlag)
10100	{
10101	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10102	AssertRCReturn(rc2, rc2);
10103	pVCpu->hm.s.fClearTrapFlag = false;
10104	pCtx->eflags.Bits.u1TF = 0;
10105	}
10106	/** @todo there seems to be issues with the resume flag when the monitor trap
10107	* flag is pending without being used. Seen early in bios init when
10108	* accessing APIC page in protected mode. */
10109
10110	/*
10111	* Restore VM-exit control settings as we may not reenter this function the
10112	* next time around.
10113	*/
10114	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10115
10116	/* Restore HMCPU indicators. */
10117	pVCpu->hm.s.fUsingDebugLoop = false;
10118	pVCpu->hm.s.fDebugWantRdTscExit = false;
10119	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10120
10121	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10122	return rcStrict;
10123	}
10124
10125
10126	/** @} */
10127
10128
10129	/**
10130	* Checks if any expensive dtrace probes are enabled and we should go to the
10131	* debug loop.
10132	*
10133	* @returns true if we should use debug loop, false if not.
10134	*/
10135	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10136	{
10137	/* It's probably faster to OR the raw 32-bit counter variables together.
10138	Since the variables are in an array and the probes are next to one
10139	another (more or less), we have good locality. So, better read
10140	eight-nine cache lines ever time and only have one conditional, than
10141	128+ conditionals, right? */
10142	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10143	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10144	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10145	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10146	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10147	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10148	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10149	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10150	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10151	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10152	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10153	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10154	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10155	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10156	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10157	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10158	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10159	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10160	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10161	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10162	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10163	) != 0
10164	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10165	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10166	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10167	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10168	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10169	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10170	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10171	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10172	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10173	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10174	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10175	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10176	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10177	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10178	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10179	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10180	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10181	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10182	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10183	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10184	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10185	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10186	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10187	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10188	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10189	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10190	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10191	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10192	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10193	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10194	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10195	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10196	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10197	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10198	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10199	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10200	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10201	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10202	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10203	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10204	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10205	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10206	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10207	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10208	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10209	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10210	) != 0
10211	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10212	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10213	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10214	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10215	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10216	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10217	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10218	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10219	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10220	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10221	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10222	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10223	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10224	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10225	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10226	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10227	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10228	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10229	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10230	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10231	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10232	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10233	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10234	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10235	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10236	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10237	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10238	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10239	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10240	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10241	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10242	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10243	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10244	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10245	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10246	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10247	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10248	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10249	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10250	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10251	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10252	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10253	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10254	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10255	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10256	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10257	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10258	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10259	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10260	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10261	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10262	) != 0;
10263	}
10264
10265
10266	/**
10267	* Runs the guest code using VT-x.
10268	*
10269	* @returns Strict VBox status code (i.e. informational status codes too).
10270	* @param pVM The cross context VM structure.
10271	* @param pVCpu The cross context virtual CPU structure.
10272	* @param pCtx Pointer to the guest-CPU context.
10273	*/
10274	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10275	{
10276	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10277	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10278	HMVMX_ASSERT_PREEMPT_SAFE();
10279
10280	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10281
10282	VBOXSTRICTRC rcStrict;
10283	if ( !pVCpu->hm.s.fUseDebugLoop
10284	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10285	&& !DBGFIsStepping(pVCpu) )
10286	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10287	else
10288	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10289
10290	if (rcStrict == VERR_EM_INTERPRETER)
10291	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10292	else if (rcStrict == VINF_EM_RESET)
10293	rcStrict = VINF_EM_TRIPLE_FAULT;
10294
10295	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10296	if (RT_FAILURE(rc2))
10297	{
10298	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10299	rcStrict = rc2;
10300	}
10301	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10302	return rcStrict;
10303	}
10304
10305
10306	#ifndef HMVMX_USE_FUNCTION_TABLE
10307	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10308	{
10309	# ifdef DEBUG_ramshankar
10310	# define RETURN_EXIT_CALL(a_CallExpr) \
10311	do { \
10312	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10313	VBOXSTRICTRC rcStrict = a_CallExpr; \
10314	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10315	return rcStrict; \
10316	} while (0)
10317	# else
10318	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10319	# endif
10320	switch (rcReason)
10321	{
10322	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10323	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10324	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10325	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10326	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10327	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10328	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10329	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10330	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10331	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10332	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10333	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10334	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10335	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10336	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10337	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10338	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10339	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10340	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10341	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10342	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10343	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10344	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10345	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10346	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10347	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10348	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10349	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10350	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10351	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10352	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10353	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10354	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10355	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10356
10357	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10358	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10359	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10360	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10361	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10362	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10363	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10364	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10365	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10366
10367	case VMX_EXIT_VMCLEAR:
10368	case VMX_EXIT_VMLAUNCH:
10369	case VMX_EXIT_VMPTRLD:
10370	case VMX_EXIT_VMPTRST:
10371	case VMX_EXIT_VMREAD:
10372	case VMX_EXIT_VMRESUME:
10373	case VMX_EXIT_VMWRITE:
10374	case VMX_EXIT_VMXOFF:
10375	case VMX_EXIT_VMXON:
10376	case VMX_EXIT_INVEPT:
10377	case VMX_EXIT_INVVPID:
10378	case VMX_EXIT_VMFUNC:
10379	case VMX_EXIT_XSAVES:
10380	case VMX_EXIT_XRSTORS:
10381	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10382	case VMX_EXIT_RESERVED_60:
10383	case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10384	case VMX_EXIT_RESERVED_62:
10385	default:
10386	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10387	}
10388	#undef RETURN_EXIT_CALL
10389	}
10390	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10391
10392
10393	#ifdef VBOX_STRICT
10394	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10395	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10396	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10397
10398	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10399	do { \
10400	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10401	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10402	} while (0)
10403
10404	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10405	do { \
10406	AssertPtr(pVCpu); \
10407	AssertPtr(pMixedCtx); \
10408	AssertPtr(pVmxTransient); \
10409	Assert(pVmxTransient->fVMEntryFailed == false); \
10410	Assert(ASMIntAreEnabled()); \
10411	HMVMX_ASSERT_PREEMPT_SAFE(); \
10412	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10413	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10414	HMVMX_ASSERT_PREEMPT_SAFE(); \
10415	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10416	HMVMX_ASSERT_PREEMPT_CPUID(); \
10417	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10418	} while (0)
10419
10420	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10421	do { \
10422	Log4Func(("\n")); \
10423	} while (0)
10424	#else /* nonstrict builds: */
10425	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10426	do { \
10427	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10428	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10429	} while (0)
10430	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10431	#endif
10432
10433
10434	/**
10435	* Advances the guest RIP by the specified number of bytes.
10436	*
10437	* @param pVCpu The cross context virtual CPU structure.
10438	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10439	* out-of-sync. Make sure to update the required fields
10440	* before using them.
10441	* @param cbInstr Number of bytes to advance the RIP by.
10442	*
10443	* @remarks No-long-jump zone!!!
10444	*/
10445	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10446	{
10447	/* Advance the RIP. */
10448	pMixedCtx->rip += cbInstr;
10449	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10450
10451	/* Update interrupt inhibition. */
10452	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10453	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10454	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10455	}
10456
10457
10458	/**
10459	* Advances the guest RIP after reading it from the VMCS.
10460	*
10461	* @returns VBox status code, no informational status codes.
10462	* @param pVCpu The cross context virtual CPU structure.
10463	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10464	* out-of-sync. Make sure to update the required fields
10465	* before using them.
10466	* @param pVmxTransient Pointer to the VMX transient structure.
10467	*
10468	* @remarks No-long-jump zone!!!
10469	*/
10470	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10471	{
10472	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10473	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10474	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10475	AssertRCReturn(rc, rc);
10476
10477	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10478
10479	/*
10480	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10481	* pending debug exception field as it takes care of priority of events.
10482	*
10483	* See Intel spec. 32.2.1 "Debug Exceptions".
10484	*/
10485	if ( !pVCpu->hm.s.fSingleInstruction
10486	&& pMixedCtx->eflags.Bits.u1TF)
10487	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10488
10489	return VINF_SUCCESS;
10490	}
10491
10492
10493	/**
10494	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10495	* and update error record fields accordingly.
10496	*
10497	* @return VMX_IGS_* return codes.
10498	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10499	* wrong with the guest state.
10500	*
10501	* @param pVM The cross context VM structure.
10502	* @param pVCpu The cross context virtual CPU structure.
10503	* @param pCtx Pointer to the guest-CPU state.
10504	*
10505	* @remarks This function assumes our cache of the VMCS controls
10506	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10507	*/
10508	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10509	{
10510	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10511	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10512	uError = (err); \
10513	break; \
10514	} else do { } while (0)
10515
10516	int rc;
10517	uint32_t uError = VMX_IGS_ERROR;
10518	uint32_t u32Val;
10519	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10520
10521	do
10522	{
10523	/*
10524	* CR0.
10525	*/
10526	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10527	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10528	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10529	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10530	if (fUnrestrictedGuest)
10531	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10532
10533	uint32_t u32GuestCR0;
10534	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10535	AssertRCBreak(rc);
10536	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10537	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10538	if ( !fUnrestrictedGuest
10539	&& (u32GuestCR0 & X86_CR0_PG)
10540	&& !(u32GuestCR0 & X86_CR0_PE))
10541	{
10542	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10543	}
10544
10545	/*
10546	* CR4.
10547	*/
10548	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10549	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10550
10551	uint32_t u32GuestCR4;
10552	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10553	AssertRCBreak(rc);
10554	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10555	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10556
10557	/*
10558	* IA32_DEBUGCTL MSR.
10559	*/
10560	uint64_t u64Val;
10561	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10562	AssertRCBreak(rc);
10563	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10564	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10565	{
10566	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10567	}
10568	uint64_t u64DebugCtlMsr = u64Val;
10569
10570	#ifdef VBOX_STRICT
10571	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10572	AssertRCBreak(rc);
10573	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10574	#endif
10575	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10576
10577	/*
10578	* RIP and RFLAGS.
10579	*/
10580	uint32_t u32Eflags;
10581	#if HC_ARCH_BITS == 64
10582	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10583	AssertRCBreak(rc);
10584	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10585	if ( !fLongModeGuest
10586	\|\| !pCtx->cs.Attr.n.u1Long)
10587	{
10588	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10589	}
10590	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10591	* must be identical if the "IA-32e mode guest" VM-entry
10592	* control is 1 and CS.L is 1. No check applies if the
10593	* CPU supports 64 linear-address bits. */
10594
10595	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10596	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10597	AssertRCBreak(rc);
10598	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10599	VMX_IGS_RFLAGS_RESERVED);
10600	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10601	u32Eflags = u64Val;
10602	#else
10603	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10604	AssertRCBreak(rc);
10605	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10606	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10607	#endif
10608
10609	if ( fLongModeGuest
10610	\|\| ( fUnrestrictedGuest
10611	&& !(u32GuestCR0 & X86_CR0_PE)))
10612	{
10613	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10614	}
10615
10616	uint32_t u32EntryInfo;
10617	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10618	AssertRCBreak(rc);
10619	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10620	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10621	{
10622	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10623	}
10624
10625	/*
10626	* 64-bit checks.
10627	*/
10628	#if HC_ARCH_BITS == 64
10629	if (fLongModeGuest)
10630	{
10631	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10632	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10633	}
10634
10635	if ( !fLongModeGuest
10636	&& (u32GuestCR4 & X86_CR4_PCIDE))
10637	{
10638	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10639	}
10640
10641	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10642	* 51:32 beyond the processor's physical-address width are 0. */
10643
10644	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10645	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10646	{
10647	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10648	}
10649
10650	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10651	AssertRCBreak(rc);
10652	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10653
10654	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10655	AssertRCBreak(rc);
10656	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10657	#endif
10658
10659	/*
10660	* PERF_GLOBAL MSR.
10661	*/
10662	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10663	{
10664	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10665	AssertRCBreak(rc);
10666	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10667	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10668	}
10669
10670	/*
10671	* PAT MSR.
10672	*/
10673	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10674	{
10675	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10676	AssertRCBreak(rc);
10677	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10678	for (unsigned i = 0; i < 8; i++)
10679	{
10680	uint8_t u8Val = (u64Val & 0xff);
10681	if ( u8Val != 0 /* UC */
10682	&& u8Val != 1 /* WC */
10683	&& u8Val != 4 /* WT */
10684	&& u8Val != 5 /* WP */
10685	&& u8Val != 6 /* WB */
10686	&& u8Val != 7 /* UC- */)
10687	{
10688	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10689	}
10690	u64Val >>= 8;
10691	}
10692	}
10693
10694	/*
10695	* EFER MSR.
10696	*/
10697	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10698	{
10699	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10700	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10701	AssertRCBreak(rc);
10702	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10703	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10704	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10705	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10706	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10707	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10708	\|\| !(u32GuestCR0 & X86_CR0_PG)
10709	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10710	VMX_IGS_EFER_LMA_LME_MISMATCH);
10711	}
10712
10713	/*
10714	* Segment registers.
10715	*/
10716	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10717	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10718	if (!(u32Eflags & X86_EFL_VM))
10719	{
10720	/* CS */
10721	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10722	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10723	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10724	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10725	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10726	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10727	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10728	/* CS cannot be loaded with NULL in protected mode. */
10729	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10730	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10731	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10732	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10733	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10734	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10735	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10736	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10737	else
10738	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10739
10740	/* SS */
10741	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10742	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10743	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10744	if ( !(pCtx->cr0 & X86_CR0_PE)
10745	\|\| pCtx->cs.Attr.n.u4Type == 3)
10746	{
10747	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10748	}
10749	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10750	{
10751	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10752	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10753	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10754	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10755	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10756	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10757	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10758	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10759	}
10760
10761	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10762	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10763	{
10764	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10765	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10766	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10767	\|\| pCtx->ds.Attr.n.u4Type > 11
10768	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10769	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10770	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10771	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10772	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10773	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10774	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10775	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10776	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10777	}
10778	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10779	{
10780	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10781	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10782	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10783	\|\| pCtx->es.Attr.n.u4Type > 11
10784	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10785	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10786	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10787	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10788	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10789	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10790	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10791	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10792	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10793	}
10794	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10795	{
10796	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10797	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10798	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10799	\|\| pCtx->fs.Attr.n.u4Type > 11
10800	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10801	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10802	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10803	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10804	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10805	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10806	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10807	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10808	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10809	}
10810	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10811	{
10812	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10813	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10814	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10815	\|\| pCtx->gs.Attr.n.u4Type > 11
10816	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10817	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10818	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10819	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10820	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10821	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10822	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10823	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10824	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10825	}
10826	/* 64-bit capable CPUs. */
10827	#if HC_ARCH_BITS == 64
10828	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10829	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10830	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10831	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10832	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10833	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
10834	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10835	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
10836	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10837	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
10838	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10839	#endif
10840	}
10841	else
10842	{
10843	/* V86 mode checks. */
10844	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10845	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10846	{
10847	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10848	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10849	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10850	}
10851	else
10852	{
10853	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10854	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10855	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10856	}
10857
10858	/* CS */
10859	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10860	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10861	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10862	/* SS */
10863	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10864	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10865	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10866	/* DS */
10867	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10868	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10869	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10870	/* ES */
10871	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10872	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10873	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10874	/* FS */
10875	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10876	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10877	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10878	/* GS */
10879	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10880	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10881	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10882	/* 64-bit capable CPUs. */
10883	#if HC_ARCH_BITS == 64
10884	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10885	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10886	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10887	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10888	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10889	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
10890	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10891	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
10892	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10893	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
10894	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10895	#endif
10896	}
10897
10898	/*
10899	* TR.
10900	*/
10901	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10902	/* 64-bit capable CPUs. */
10903	#if HC_ARCH_BITS == 64
10904	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10905	#endif
10906	if (fLongModeGuest)
10907	{
10908	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10909	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10910	}
10911	else
10912	{
10913	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10914	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10915	VMX_IGS_TR_ATTR_TYPE_INVALID);
10916	}
10917	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10918	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10919	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10920	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10921	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10922	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10923	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10924	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10925
10926	/*
10927	* GDTR and IDTR.
10928	*/
10929	#if HC_ARCH_BITS == 64
10930	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10931	AssertRCBreak(rc);
10932	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10933
10934	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10935	AssertRCBreak(rc);
10936	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10937	#endif
10938
10939	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10940	AssertRCBreak(rc);
10941	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10942
10943	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10944	AssertRCBreak(rc);
10945	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10946
10947	/*
10948	* Guest Non-Register State.
10949	*/
10950	/* Activity State. */
10951	uint32_t u32ActivityState;
10952	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10953	AssertRCBreak(rc);
10954	HMVMX_CHECK_BREAK( !u32ActivityState
10955	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10956	VMX_IGS_ACTIVITY_STATE_INVALID);
10957	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10958	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10959	uint32_t u32IntrState;
10960	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10961	AssertRCBreak(rc);
10962	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10963	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10964	{
10965	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10966	}
10967
10968	/** @todo Activity state and injecting interrupts. Left as a todo since we
10969	* currently don't use activity states but ACTIVE. */
10970
10971	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10972	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10973
10974	/* Guest interruptibility-state. */
10975	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10976	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10977	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10978	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10979	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10980	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10981	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10982	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10983	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10984	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10985	{
10986	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10987	{
10988	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10989	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10990	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10991	}
10992	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10993	{
10994	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10995	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10996	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10997	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10998	}
10999	}
11000	/** @todo Assumes the processor is not in SMM. */
11001	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11002	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11003	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11004	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11005	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11006	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11007	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11008	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11009	{
11010	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11011	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11012	}
11013
11014	/* Pending debug exceptions. */
11015	#if HC_ARCH_BITS == 64
11016	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11017	AssertRCBreak(rc);
11018	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11019	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11020	u32Val = u64Val; /* For pending debug exceptions checks below. */
11021	#else
11022	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11023	AssertRCBreak(rc);
11024	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11025	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11026	#endif
11027
11028	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11029	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11030	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11031	{
11032	if ( (u32Eflags & X86_EFL_TF)
11033	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11034	{
11035	/* Bit 14 is PendingDebug.BS. */
11036	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11037	}
11038	if ( !(u32Eflags & X86_EFL_TF)
11039	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11040	{
11041	/* Bit 14 is PendingDebug.BS. */
11042	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11043	}
11044	}
11045
11046	/* VMCS link pointer. */
11047	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11048	AssertRCBreak(rc);
11049	if (u64Val != UINT64_C(0xffffffffffffffff))
11050	{
11051	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11052	/** @todo Bits beyond the processor's physical-address width MBZ. */
11053	/** @todo 32-bit located in memory referenced by value of this field (as a
11054	* physical address) must contain the processor's VMCS revision ID. */
11055	/** @todo SMM checks. */
11056	}
11057
11058	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11059	* not using Nested Paging? */
11060	if ( pVM->hm.s.fNestedPaging
11061	&& !fLongModeGuest
11062	&& CPUMIsGuestInPAEModeEx(pCtx))
11063	{
11064	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11065	AssertRCBreak(rc);
11066	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11067
11068	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11069	AssertRCBreak(rc);
11070	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11071
11072	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11073	AssertRCBreak(rc);
11074	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11075
11076	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11077	AssertRCBreak(rc);
11078	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11079	}
11080
11081	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11082	if (uError == VMX_IGS_ERROR)
11083	uError = VMX_IGS_REASON_NOT_FOUND;
11084	} while (0);
11085
11086	pVCpu->hm.s.u32HMError = uError;
11087	return uError;
11088
11089	#undef HMVMX_ERROR_BREAK
11090	#undef HMVMX_CHECK_BREAK
11091	}
11092
11093	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11094	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11095	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11096
11097	/** @name VM-exit handlers.
11098	* @{
11099	*/
11100
11101	/**
11102	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11103	*/
11104	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11105	{
11106	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11107	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11108	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11109	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11110	return VINF_SUCCESS;
11111	return VINF_EM_RAW_INTERRUPT;
11112	}
11113
11114
11115	/**
11116	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11117	*/
11118	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11119	{
11120	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11121	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11122
11123	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11124	AssertRCReturn(rc, rc);
11125
11126	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11127	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11128	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11129	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11130
11131	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11132	{
11133	/*
11134	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11135	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11136	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11137	*
11138	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11139	*/
11140	VMXDispatchHostNmi();
11141	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11142	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11143	return VINF_SUCCESS;
11144	}
11145
11146	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11147	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11148	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11149	{ /* likely */ }
11150	else
11151	{
11152	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11153	rcStrictRc1 = VINF_SUCCESS;
11154	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11155	return rcStrictRc1;
11156	}
11157
11158	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11159	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11160	switch (uIntType)
11161	{
11162	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11163	Assert(uVector == X86_XCPT_DB);
11164	/* no break */
11165	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11166	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11167	/* no break */
11168	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11169	{
11170	switch (uVector)
11171	{
11172	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11173	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11174	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11175	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11176	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11177	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11178	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11179
11180	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11181	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11182	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11183	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11184	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11185	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11186	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11187	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11188	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11189	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11190	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11191	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11192	default:
11193	{
11194	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11195	AssertRCReturn(rc, rc);
11196
11197	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11198	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11199	{
11200	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11201	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11202	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11203
11204	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11205	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11206	AssertRCReturn(rc, rc);
11207	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11208	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11209	0 /* GCPtrFaultAddress */);
11210	AssertRCReturn(rc, rc);
11211	}
11212	else
11213	{
11214	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11215	pVCpu->hm.s.u32HMError = uVector;
11216	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11217	}
11218	break;
11219	}
11220	}
11221	break;
11222	}
11223
11224	default:
11225	{
11226	pVCpu->hm.s.u32HMError = uExitIntInfo;
11227	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11228	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11229	break;
11230	}
11231	}
11232	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11233	return rc;
11234	}
11235
11236
11237	/**
11238	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11239	*/
11240	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11241	{
11242	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11243
11244	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11245	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11246
11247	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11248	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11249	return VINF_SUCCESS;
11250	}
11251
11252
11253	/**
11254	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11255	*/
11256	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11257	{
11258	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11259	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11260	{
11261	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11262	HMVMX_RETURN_UNEXPECTED_EXIT();
11263	}
11264
11265	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11266
11267	/*
11268	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11269	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11270	*/
11271	uint32_t uIntrState = 0;
11272	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11273	AssertRCReturn(rc, rc);
11274
11275	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11276	if ( fBlockSti
11277	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11278	{
11279	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11280	}
11281
11282	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11283	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11284
11285	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11286	return VINF_SUCCESS;
11287	}
11288
11289
11290	/**
11291	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11292	*/
11293	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11294	{
11295	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11296	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11297	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11298	}
11299
11300
11301	/**
11302	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11303	*/
11304	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11305	{
11306	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11307	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11308	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11309	}
11310
11311
11312	/**
11313	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11314	*/
11315	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11316	{
11317	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11318	PVM pVM = pVCpu->CTX_SUFF(pVM);
11319	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11320	if (RT_LIKELY(rc == VINF_SUCCESS))
11321	{
11322	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11323	Assert(pVmxTransient->cbInstr == 2);
11324	}
11325	else
11326	{
11327	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11328	rc = VERR_EM_INTERPRETER;
11329	}
11330	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11331	return rc;
11332	}
11333
11334
11335	/**
11336	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11337	*/
11338	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11339	{
11340	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11341	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11342	AssertRCReturn(rc, rc);
11343
11344	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11345	return VINF_EM_RAW_EMULATE_INSTR;
11346
11347	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11348	HMVMX_RETURN_UNEXPECTED_EXIT();
11349	}
11350
11351
11352	/**
11353	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11354	*/
11355	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11356	{
11357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11358	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11359	AssertRCReturn(rc, rc);
11360
11361	PVM pVM = pVCpu->CTX_SUFF(pVM);
11362	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11363	if (RT_LIKELY(rc == VINF_SUCCESS))
11364	{
11365	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11366	Assert(pVmxTransient->cbInstr == 2);
11367	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11368	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11369	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11370	}
11371	else
11372	rc = VERR_EM_INTERPRETER;
11373	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11374	return rc;
11375	}
11376
11377
11378	/**
11379	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11380	*/
11381	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11382	{
11383	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11384	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11385	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11386	AssertRCReturn(rc, rc);
11387
11388	PVM pVM = pVCpu->CTX_SUFF(pVM);
11389	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11390	if (RT_SUCCESS(rc))
11391	{
11392	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11393	Assert(pVmxTransient->cbInstr == 3);
11394	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11395	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11396	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11397	}
11398	else
11399	{
11400	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11401	rc = VERR_EM_INTERPRETER;
11402	}
11403	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11404	return rc;
11405	}
11406
11407
11408	/**
11409	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11410	*/
11411	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11412	{
11413	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11414	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11415	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
11416	AssertRCReturn(rc, rc);
11417
11418	PVM pVM = pVCpu->CTX_SUFF(pVM);
11419	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11420	if (RT_LIKELY(rc == VINF_SUCCESS))
11421	{
11422	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11423	Assert(pVmxTransient->cbInstr == 2);
11424	}
11425	else
11426	{
11427	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11428	rc = VERR_EM_INTERPRETER;
11429	}
11430	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11431	return rc;
11432	}
11433
11434
11435	/**
11436	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11437	*/
11438	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11439	{
11440	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11441	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11442
11443	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11444	if (pVCpu->hm.s.fHypercallsEnabled)
11445	{
11446	#if 0
11447	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11448	#else
11449	/* Aggressive state sync. for now. */
11450	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11451	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11452	AssertRCReturn(rc, rc);
11453	#endif
11454
11455	/* Perform the hypercall. */
11456	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11457	if (rcStrict == VINF_SUCCESS)
11458	{
11459	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11460	AssertRCReturn(rc, rc);
11461	}
11462	else
11463	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11464	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11465	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11466
11467	/* If the hypercall changes anything other than guest's general-purpose registers,
11468	we would need to reload the guest changed bits here before VM-entry. */
11469	}
11470	else
11471	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11472
11473	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11474	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11475	{
11476	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11477	rcStrict = VINF_SUCCESS;
11478	}
11479
11480	return rcStrict;
11481	}
11482
11483
11484	/**
11485	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11486	*/
11487	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11488	{
11489	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11490	PVM pVM = pVCpu->CTX_SUFF(pVM);
11491	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11492
11493	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11494	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11495	AssertRCReturn(rc, rc);
11496
11497	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11498	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11499	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11500	else
11501	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11502	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11504	return rcStrict;
11505	}
11506
11507
11508	/**
11509	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11510	*/
11511	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11512	{
11513	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11514	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11515	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11516	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11517	AssertRCReturn(rc, rc);
11518
11519	PVM pVM = pVCpu->CTX_SUFF(pVM);
11520	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11521	if (RT_LIKELY(rc == VINF_SUCCESS))
11522	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11523	else
11524	{
11525	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11526	rc = VERR_EM_INTERPRETER;
11527	}
11528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11529	return rc;
11530	}
11531
11532
11533	/**
11534	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11535	*/
11536	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11537	{
11538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11539	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11540	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11541	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11542	AssertRCReturn(rc, rc);
11543
11544	PVM pVM = pVCpu->CTX_SUFF(pVM);
11545	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11546	rc = VBOXSTRICTRC_VAL(rc2);
11547	if (RT_LIKELY( rc == VINF_SUCCESS
11548	\|\| rc == VINF_EM_HALT))
11549	{
11550	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11551	AssertRCReturn(rc3, rc3);
11552
11553	if ( rc == VINF_EM_HALT
11554	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11555	{
11556	rc = VINF_SUCCESS;
11557	}
11558	}
11559	else
11560	{
11561	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11562	rc = VERR_EM_INTERPRETER;
11563	}
11564	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11565	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11566	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11567	return rc;
11568	}
11569
11570
11571	/**
11572	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11573	*/
11574	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11575	{
11576	/*
11577	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11578	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11579	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
11580	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11581	*/
11582	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11583	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11584	HMVMX_RETURN_UNEXPECTED_EXIT();
11585	}
11586
11587
11588	/**
11589	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11590	*/
11591	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11592	{
11593	/*
11594	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11595	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11596	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
11597	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11598	*/
11599	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11600	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11601	HMVMX_RETURN_UNEXPECTED_EXIT();
11602	}
11603
11604
11605	/**
11606	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11607	*/
11608	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11609	{
11610	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11611	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11612	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11613	HMVMX_RETURN_UNEXPECTED_EXIT();
11614	}
11615
11616
11617	/**
11618	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11619	*/
11620	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11621	{
11622	/*
11623	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11624	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11625	* See Intel spec. 25.3 "Other Causes of VM-exits".
11626	*/
11627	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11628	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11629	HMVMX_RETURN_UNEXPECTED_EXIT();
11630	}
11631
11632
11633	/**
11634	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11635	* VM-exit.
11636	*/
11637	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11638	{
11639	/*
11640	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11641	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11642	*
11643	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11644	* See Intel spec. "23.8 Restrictions on VMX operation".
11645	*/
11646	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11647	return VINF_SUCCESS;
11648	}
11649
11650
11651	/**
11652	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11653	* VM-exit.
11654	*/
11655	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11656	{
11657	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11658	return VINF_EM_RESET;
11659	}
11660
11661
11662	/**
11663	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11664	*/
11665	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11666	{
11667	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11668	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11669
11670	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11671	AssertRCReturn(rc, rc);
11672
11673	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11674	rc = VINF_SUCCESS;
11675	else
11676	rc = VINF_EM_HALT;
11677
11678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11679	if (rc != VINF_SUCCESS)
11680	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11681	return rc;
11682	}
11683
11684
11685	/**
11686	* VM-exit handler for instructions that result in a \#UD exception delivered to
11687	* the guest.
11688	*/
11689	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11690	{
11691	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11692	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11693	return VINF_SUCCESS;
11694	}
11695
11696
11697	/**
11698	* VM-exit handler for expiry of the VMX preemption timer.
11699	*/
11700	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11701	{
11702	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11703
11704	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11705	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11706
11707	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11708	PVM pVM = pVCpu->CTX_SUFF(pVM);
11709	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11710	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11711	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11712	}
11713
11714
11715	/**
11716	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11717	*/
11718	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11719	{
11720	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11721
11722	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11723	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
11724	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11725	AssertRCReturn(rc, rc);
11726
11727	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11728	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11729
11730	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11731
11732	return rcStrict;
11733	}
11734
11735
11736	/**
11737	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11738	*/
11739	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11740	{
11741	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11742
11743	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11744	/** @todo implement EMInterpretInvpcid() */
11745	return VERR_EM_INTERPRETER;
11746	}
11747
11748
11749	/**
11750	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11751	* Error VM-exit.
11752	*/
11753	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11754	{
11755	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11756	AssertRCReturn(rc, rc);
11757
11758	rc = hmR0VmxCheckVmcsCtls(pVCpu);
11759	AssertRCReturn(rc, rc);
11760
11761	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11762	NOREF(uInvalidReason);
11763
11764	#ifdef VBOX_STRICT
11765	uint32_t uIntrState;
11766	RTHCUINTREG uHCReg;
11767	uint64_t u64Val;
11768	uint32_t u32Val;
11769
11770	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11771	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11772	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11773	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11774	AssertRCReturn(rc, rc);
11775
11776	Log4(("uInvalidReason %u\n", uInvalidReason));
11777	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11778	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11779	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11780	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11781
11782	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11783	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11784	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11785	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11786	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11787	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11788	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11789	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11790	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11791	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11792	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11793	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11794	#else
11795	NOREF(pVmxTransient);
11796	#endif
11797
11798	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11799	return VERR_VMX_INVALID_GUEST_STATE;
11800	}
11801
11802
11803	/**
11804	* VM-exit handler for VM-entry failure due to an MSR-load
11805	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11806	*/
11807	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11808	{
11809	NOREF(pVmxTransient);
11810	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11811	HMVMX_RETURN_UNEXPECTED_EXIT();
11812	}
11813
11814
11815	/**
11816	* VM-exit handler for VM-entry failure due to a machine-check event
11817	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11818	*/
11819	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11820	{
11821	NOREF(pVmxTransient);
11822	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11823	HMVMX_RETURN_UNEXPECTED_EXIT();
11824	}
11825
11826
11827	/**
11828	* VM-exit handler for all undefined reasons. Should never ever happen.. in
11829	* theory.
11830	*/
11831	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11832	{
11833	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11834	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11835	return VERR_VMX_UNDEFINED_EXIT_CODE;
11836	}
11837
11838
11839	/**
11840	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11841	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11842	* Conditional VM-exit.
11843	*/
11844	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11845	{
11846	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11847
11848	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11849	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11850	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11851	return VERR_EM_INTERPRETER;
11852	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11853	HMVMX_RETURN_UNEXPECTED_EXIT();
11854	}
11855
11856
11857	/**
11858	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11859	*/
11860	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11861	{
11862	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11863
11864	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11865	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11866	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11867	return VERR_EM_INTERPRETER;
11868	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11869	HMVMX_RETURN_UNEXPECTED_EXIT();
11870	}
11871
11872
11873	/**
11874	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11875	*/
11876	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11877	{
11878	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11879
11880	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11881	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11882	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11883	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11884	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11885	{
11886	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11887	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11888	}
11889	AssertRCReturn(rc, rc);
11890	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11891
11892	#ifdef VBOX_STRICT
11893	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11894	{
11895	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11896	&& pMixedCtx->ecx != MSR_K6_EFER)
11897	{
11898	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11899	pMixedCtx->ecx));
11900	HMVMX_RETURN_UNEXPECTED_EXIT();
11901	}
11902	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11903	{
11904	VMXMSREXITREAD enmRead;
11905	VMXMSREXITWRITE enmWrite;
11906	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11907	AssertRCReturn(rc2, rc2);
11908	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11909	{
11910	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11911	HMVMX_RETURN_UNEXPECTED_EXIT();
11912	}
11913	}
11914	}
11915	#endif
11916
11917	PVM pVM = pVCpu->CTX_SUFF(pVM);
11918	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11919	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
11920	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11921	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11922	if (RT_SUCCESS(rc))
11923	{
11924	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11925	Assert(pVmxTransient->cbInstr == 2);
11926	}
11927	return rc;
11928	}
11929
11930
11931	/**
11932	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11933	*/
11934	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11935	{
11936	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11937	PVM pVM = pVCpu->CTX_SUFF(pVM);
11938	int rc = VINF_SUCCESS;
11939
11940	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11941	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11942	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11943	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11944	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11945	{
11946	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11947	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11948	}
11949	AssertRCReturn(rc, rc);
11950	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11951
11952	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11953	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11955
11956	if (RT_SUCCESS(rc))
11957	{
11958	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11959
11960	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
11961	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11962	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
11963	{
11964	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11965	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11966	EMInterpretWrmsr() changes it. */
11967	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11968	}
11969	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11970	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11971	else if (pMixedCtx->ecx == MSR_K6_EFER)
11972	{
11973	/*
11974	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11975	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11976	* the other bits as well, SCE and NXE. See @bugref{7368}.
11977	*/
11978	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
11979	}
11980
11981	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11982	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11983	{
11984	switch (pMixedCtx->ecx)
11985	{
11986	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11987	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11988	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11989	case MSR_K8_FS_BASE: /* no break */
11990	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
11991	case MSR_K6_EFER: /* already handled above */ break;
11992	default:
11993	{
11994	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11995	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11996	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11997	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
11998	break;
11999	}
12000	}
12001	}
12002	#ifdef VBOX_STRICT
12003	else
12004	{
12005	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12006	switch (pMixedCtx->ecx)
12007	{
12008	case MSR_IA32_SYSENTER_CS:
12009	case MSR_IA32_SYSENTER_EIP:
12010	case MSR_IA32_SYSENTER_ESP:
12011	case MSR_K8_FS_BASE:
12012	case MSR_K8_GS_BASE:
12013	{
12014	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12015	HMVMX_RETURN_UNEXPECTED_EXIT();
12016	}
12017
12018	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12019	default:
12020	{
12021	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12022	{
12023	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12024	if (pMixedCtx->ecx != MSR_K6_EFER)
12025	{
12026	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12027	pMixedCtx->ecx));
12028	HMVMX_RETURN_UNEXPECTED_EXIT();
12029	}
12030	}
12031
12032	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12033	{
12034	VMXMSREXITREAD enmRead;
12035	VMXMSREXITWRITE enmWrite;
12036	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12037	AssertRCReturn(rc2, rc2);
12038	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12039	{
12040	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12041	HMVMX_RETURN_UNEXPECTED_EXIT();
12042	}
12043	}
12044	break;
12045	}
12046	}
12047	}
12048	#endif /* VBOX_STRICT */
12049	}
12050	return rc;
12051	}
12052
12053
12054	/**
12055	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12056	*/
12057	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12058	{
12059	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12060
12061	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12062	return VINF_EM_RAW_INTERRUPT;
12063	}
12064
12065
12066	/**
12067	* VM-exit handler for when the TPR value is lowered below the specified
12068	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12069	*/
12070	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12071	{
12072	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12073	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12074
12075	/*
12076	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12077	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12078	* resume guest execution.
12079	*/
12080	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12081	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12082	return VINF_SUCCESS;
12083	}
12084
12085
12086	/**
12087	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12088	* VM-exit.
12089	*
12090	* @retval VINF_SUCCESS when guest execution can continue.
12091	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12092	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12093	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12094	* interpreter.
12095	*/
12096	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12097	{
12098	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12099	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12100	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12101	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12102	AssertRCReturn(rc, rc);
12103
12104	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12105	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12106	PVM pVM = pVCpu->CTX_SUFF(pVM);
12107	VBOXSTRICTRC rcStrict;
12108	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12109	switch (uAccessType)
12110	{
12111	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12112	{
12113	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12114	AssertRCReturn(rc, rc);
12115
12116	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12117	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12118	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12119	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12120	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12121	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12122	{
12123	case 0: /* CR0 */
12124	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12125	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12126	break;
12127	case 2: /* CR2 */
12128	/* Nothing to do here, CR2 it's not part of the VMCS. */
12129	break;
12130	case 3: /* CR3 */
12131	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12132	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12133	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12134	break;
12135	case 4: /* CR4 */
12136	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12137	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12138	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12139	break;
12140	case 8: /* CR8 */
12141	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12142	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12143	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12144	break;
12145	default:
12146	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12147	break;
12148	}
12149
12150	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12151	break;
12152	}
12153
12154	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12155	{
12156	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12157	AssertRCReturn(rc, rc);
12158
12159	Assert( !pVM->hm.s.fNestedPaging
12160	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12161	\|\| pVCpu->hm.s.fUsingDebugLoop
12162	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12163
12164	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12165	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12166	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12167
12168	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12169	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12170	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12171	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12172	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12173	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12174	VBOXSTRICTRC_VAL(rcStrict)));
12175	break;
12176	}
12177
12178	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12179	{
12180	AssertRCReturn(rc, rc);
12181	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12182	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12183	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12185	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12186	break;
12187	}
12188
12189	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12190	{
12191	AssertRCReturn(rc, rc);
12192	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12193	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12194	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12195	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12196	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12197	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12198	break;
12199	}
12200
12201	default:
12202	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12203	VERR_VMX_UNEXPECTED_EXCEPTION);
12204	}
12205
12206	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12207	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12208	NOREF(pVM);
12209	return rcStrict;
12210	}
12211
12212
12213	/**
12214	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12215	* VM-exit.
12216	*/
12217	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12218	{
12219	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12220	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12221
12222	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12223	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12224	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12225	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12226	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12227	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12228	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12229	AssertRCReturn(rc2, rc2);
12230
12231	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12232	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12233	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12234	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12235	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12236	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12237	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12238	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12239	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12240
12241	/* I/O operation lookup arrays. */
12242	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12243	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12244
12245	VBOXSTRICTRC rcStrict;
12246	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12247	uint32_t const cbInstr = pVmxTransient->cbInstr;
12248	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12249	PVM pVM = pVCpu->CTX_SUFF(pVM);
12250	if (fIOString)
12251	{
12252	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12253	See @bugref{5752#c158}. Should work now. */
12254	/*
12255	* INS/OUTS - I/O String instruction.
12256	*
12257	* Use instruction-information if available, otherwise fall back on
12258	* interpreting the instruction.
12259	*/
12260	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12261	fIOWrite ? 'w' : 'r'));
12262	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12263	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12264	{
12265	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12266	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12267	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12268	AssertRCReturn(rc2, rc2);
12269	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12270	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12271	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12272	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12273	if (fIOWrite)
12274	{
12275	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12276	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12277	}
12278	else
12279	{
12280	/*
12281	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12282	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12283	* See Intel Instruction spec. for "INS".
12284	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12285	*/
12286	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12287	}
12288	}
12289	else
12290	{
12291	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12292	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12293	AssertRCReturn(rc2, rc2);
12294	rcStrict = IEMExecOne(pVCpu);
12295	}
12296	/** @todo IEM needs to be setting these flags somehow. */
12297	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12298	fUpdateRipAlready = true;
12299	#else
12300	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12301	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12302	if (RT_SUCCESS(rcStrict))
12303	{
12304	if (fIOWrite)
12305	{
12306	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12307	(DISCPUMODE)pDis->uAddrMode, cbValue);
12308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12309	}
12310	else
12311	{
12312	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12313	(DISCPUMODE)pDis->uAddrMode, cbValue);
12314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12315	}
12316	}
12317	else
12318	{
12319	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12320	pMixedCtx->rip));
12321	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12322	}
12323	#endif
12324	}
12325	else
12326	{
12327	/*
12328	* IN/OUT - I/O instruction.
12329	*/
12330	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12331	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12332	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12333	if (fIOWrite)
12334	{
12335	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12337	}
12338	else
12339	{
12340	uint32_t u32Result = 0;
12341	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12342	if (IOM_SUCCESS(rcStrict))
12343	{
12344	/* Save result of I/O IN instr. in AL/AX/EAX. */
12345	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12346	}
12347	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12348	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12350	}
12351	}
12352
12353	if (IOM_SUCCESS(rcStrict))
12354	{
12355	if (!fUpdateRipAlready)
12356	{
12357	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12358	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12359	}
12360
12361	/*
12362	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12363	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12364	*/
12365	if (fIOString)
12366	{
12367	/** @todo Single-step for INS/OUTS with REP prefix? */
12368	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12369	}
12370	else if ( !fDbgStepping
12371	&& fGstStepping)
12372	{
12373	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12374	}
12375
12376	/*
12377	* If any I/O breakpoints are armed, we need to check if one triggered
12378	* and take appropriate action.
12379	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12380	*/
12381	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12382	AssertRCReturn(rc2, rc2);
12383
12384	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12385	* execution engines about whether hyper BPs and such are pending. */
12386	uint32_t const uDr7 = pMixedCtx->dr[7];
12387	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12388	&& X86_DR7_ANY_RW_IO(uDr7)
12389	&& (pMixedCtx->cr4 & X86_CR4_DE))
12390	\|\| DBGFBpIsHwIoArmed(pVM)))
12391	{
12392	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12393
12394	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12395	VMMRZCallRing3Disable(pVCpu);
12396	HM_DISABLE_PREEMPT();
12397
12398	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12399
12400	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12401	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12402	{
12403	/* Raise #DB. */
12404	if (fIsGuestDbgActive)
12405	ASMSetDR6(pMixedCtx->dr[6]);
12406	if (pMixedCtx->dr[7] != uDr7)
12407	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12408
12409	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12410	}
12411	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12412	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12413	else if ( rcStrict2 != VINF_SUCCESS
12414	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12415	rcStrict = rcStrict2;
12416	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12417
12418	HM_RESTORE_PREEMPT();
12419	VMMRZCallRing3Enable(pVCpu);
12420	}
12421	}
12422
12423	#ifdef VBOX_STRICT
12424	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12425	Assert(!fIOWrite);
12426	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12427	Assert(fIOWrite);
12428	else
12429	{
12430	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12431	* statuses, that the VMM device and some others may return. See
12432	* IOM_SUCCESS() for guidance. */
12433	AssertMsg( RT_FAILURE(rcStrict)
12434	\|\| rcStrict == VINF_SUCCESS
12435	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12436	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12437	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12438	\|\| rcStrict == VINF_EM_RAW_TO_R3
12439	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12440	#endif
12441	}
12442	#endif
12443
12444	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12445	return rcStrict;
12446	}
12447
12448
12449	/**
12450	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12451	* VM-exit.
12452	*/
12453	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12454	{
12455	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12456
12457	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12458	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12459	AssertRCReturn(rc, rc);
12460	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12461	{
12462	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12463	AssertRCReturn(rc, rc);
12464	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12465	{
12466	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12467
12468	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12469	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12470
12471	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12472	Assert(!pVCpu->hm.s.Event.fPending);
12473	pVCpu->hm.s.Event.fPending = true;
12474	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12475	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12476	AssertRCReturn(rc, rc);
12477	if (fErrorCodeValid)
12478	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12479	else
12480	pVCpu->hm.s.Event.u32ErrCode = 0;
12481	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12482	&& uVector == X86_XCPT_PF)
12483	{
12484	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12485	}
12486
12487	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12488	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12489	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12490	}
12491	}
12492
12493	/** @todo Emulate task switch someday, currently just going back to ring-3 for
12494	* emulation. */
12495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12496	return VERR_EM_INTERPRETER;
12497	}
12498
12499
12500	/**
12501	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12502	*/
12503	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12504	{
12505	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12506	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12507	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12508	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12509	AssertRCReturn(rc, rc);
12510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12511	return VINF_EM_DBG_STEPPED;
12512	}
12513
12514
12515	/**
12516	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12517	*/
12518	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12519	{
12520	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12521
12522	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12523	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12524	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12525	{ /* likely */ }
12526	else
12527	{
12528	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12529	rcStrict1 = VINF_SUCCESS;
12530	return rcStrict1;
12531	}
12532
12533	#if 0
12534	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12535	* just sync the whole thing. */
12536	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12537	#else
12538	/* Aggressive state sync. for now. */
12539	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12540	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12541	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12542	#endif
12543	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12544	AssertRCReturn(rc, rc);
12545
12546	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12547	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12548	VBOXSTRICTRC rcStrict2;
12549	switch (uAccessType)
12550	{
12551	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12552	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12553	{
12554	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12555	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12556	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12557
12558	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12559	GCPhys &= PAGE_BASE_GC_MASK;
12560	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12561	PVM pVM = pVCpu->CTX_SUFF(pVM);
12562	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12563	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12564
12565	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12566	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12567	CPUMCTX2CORE(pMixedCtx), GCPhys);
12568	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12569	if ( rcStrict2 == VINF_SUCCESS
12570	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12571	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12572	{
12573	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12574	\| HM_CHANGED_GUEST_RSP
12575	\| HM_CHANGED_GUEST_RFLAGS
12576	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
12577	rcStrict2 = VINF_SUCCESS;
12578	}
12579	break;
12580	}
12581
12582	default:
12583	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12584	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12585	break;
12586	}
12587
12588	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12589	if (rcStrict2 != VINF_SUCCESS)
12590	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12591	return rcStrict2;
12592	}
12593
12594
12595	/**
12596	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12597	* VM-exit.
12598	*/
12599	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12600	{
12601	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12602
12603	/* We should -not- get this VM-exit if the guest's debug registers were active. */
12604	if (pVmxTransient->fWasGuestDebugStateActive)
12605	{
12606	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12607	HMVMX_RETURN_UNEXPECTED_EXIT();
12608	}
12609
12610	if ( !pVCpu->hm.s.fSingleInstruction
12611	&& !pVmxTransient->fWasHyperDebugStateActive)
12612	{
12613	Assert(!DBGFIsStepping(pVCpu));
12614	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12615
12616	/* Don't intercept MOV DRx any more. */
12617	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12618	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12619	AssertRCReturn(rc, rc);
12620
12621	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12622	VMMRZCallRing3Disable(pVCpu);
12623	HM_DISABLE_PREEMPT();
12624
12625	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12626	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12627	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
12628
12629	HM_RESTORE_PREEMPT();
12630	VMMRZCallRing3Enable(pVCpu);
12631
12632	#ifdef VBOX_WITH_STATISTICS
12633	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12634	AssertRCReturn(rc, rc);
12635	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12636	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12637	else
12638	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12639	#endif
12640	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12641	return VINF_SUCCESS;
12642	}
12643
12644	/*
12645	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12646	* Update the segment registers and DR7 from the CPU.
12647	*/
12648	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12649	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12650	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12651	AssertRCReturn(rc, rc);
12652	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12653
12654	PVM pVM = pVCpu->CTX_SUFF(pVM);
12655	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12656	{
12657	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12658	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12659	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12660	if (RT_SUCCESS(rc))
12661	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12663	}
12664	else
12665	{
12666	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12667	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12668	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12670	}
12671
12672	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
12673	if (RT_SUCCESS(rc))
12674	{
12675	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12676	AssertRCReturn(rc2, rc2);
12677	return VINF_SUCCESS;
12678	}
12679	return rc;
12680	}
12681
12682
12683	/**
12684	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12685	* Conditional VM-exit.
12686	*/
12687	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12688	{
12689	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12690	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12691
12692	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12693	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12694	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12695	{ /* likely */ }
12696	else
12697	{
12698	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12699	rcStrict1 = VINF_SUCCESS;
12700	return rcStrict1;
12701	}
12702
12703	RTGCPHYS GCPhys = 0;
12704	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12705
12706	#if 0
12707	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12708	#else
12709	/* Aggressive state sync. for now. */
12710	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12711	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12712	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12713	#endif
12714	AssertRCReturn(rc, rc);
12715
12716	/*
12717	* If we succeed, resume guest execution.
12718	* If we fail in interpreting the instruction because we couldn't get the guest physical address
12719	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12720	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12721	* weird case. See @bugref{6043}.
12722	*/
12723	PVM pVM = pVCpu->CTX_SUFF(pVM);
12724	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12725	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12726	if ( rcStrict2 == VINF_SUCCESS
12727	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12728	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12729	{
12730	/* Successfully handled MMIO operation. */
12731	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12732	\| HM_CHANGED_GUEST_RSP
12733	\| HM_CHANGED_GUEST_RFLAGS
12734	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
12735	return VINF_SUCCESS;
12736	}
12737	return rcStrict2;
12738	}
12739
12740
12741	/**
12742	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12743	* VM-exit.
12744	*/
12745	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12746	{
12747	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12748	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12749
12750	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12751	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12752	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12753	{ /* likely */ }
12754	else
12755	{
12756	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12757	rcStrict1 = VINF_SUCCESS;
12758	return rcStrict1;
12759	}
12760
12761	RTGCPHYS GCPhys = 0;
12762	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12763	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12764	#if 0
12765	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12766	#else
12767	/* Aggressive state sync. for now. */
12768	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12769	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12770	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12771	#endif
12772	AssertRCReturn(rc, rc);
12773
12774	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12775	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12776
12777	RTGCUINT uErrorCode = 0;
12778	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12779	uErrorCode \|= X86_TRAP_PF_ID;
12780	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12781	uErrorCode \|= X86_TRAP_PF_RW;
12782	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12783	uErrorCode \|= X86_TRAP_PF_P;
12784
12785	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12786
12787	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12788	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12789
12790	/* Handle the pagefault trap for the nested shadow table. */
12791	PVM pVM = pVCpu->CTX_SUFF(pVM);
12792	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12793	TRPMResetTrap(pVCpu);
12794
12795	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12796	if ( rcStrict2 == VINF_SUCCESS
12797	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12798	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12799	{
12800	/* Successfully synced our nested page tables. */
12801	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12802	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12803	\| HM_CHANGED_GUEST_RSP
12804	\| HM_CHANGED_GUEST_RFLAGS);
12805	return VINF_SUCCESS;
12806	}
12807
12808	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12809	return rcStrict2;
12810	}
12811
12812	/** @} */
12813
12814	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12815	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12816	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12817
12818	/** @name VM-exit exception handlers.
12819	* @{
12820	*/
12821
12822	/**
12823	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
12824	*/
12825	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12826	{
12827	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12828	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12829
12830	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12831	AssertRCReturn(rc, rc);
12832
12833	if (!(pMixedCtx->cr0 & X86_CR0_NE))
12834	{
12835	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12836	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12837
12838	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12839	* provides VM-exit instruction length. If this causes problem later,
12840	* disassemble the instruction like it's done on AMD-V. */
12841	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12842	AssertRCReturn(rc2, rc2);
12843	return rc;
12844	}
12845
12846	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12847	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12848	return rc;
12849	}
12850
12851
12852	/**
12853	* VM-exit exception handler for \#BP (Breakpoint exception).
12854	*/
12855	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12856	{
12857	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12858	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12859
12860	/** @todo Try optimize this by not saving the entire guest state unless
12861	* really needed. */
12862	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12863	AssertRCReturn(rc, rc);
12864
12865	PVM pVM = pVCpu->CTX_SUFF(pVM);
12866	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12867	if (rc == VINF_EM_RAW_GUEST_TRAP)
12868	{
12869	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12870	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12871	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12872	AssertRCReturn(rc, rc);
12873
12874	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12875	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12876	}
12877
12878	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
12879	return rc;
12880	}
12881
12882
12883	/**
12884	* VM-exit exception handler for \#AC (alignment check exception).
12885	*/
12886	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12887	{
12888	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12889
12890	/*
12891	* Re-inject it. We'll detect any nesting before getting here.
12892	*/
12893	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12894	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12895	AssertRCReturn(rc, rc);
12896	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12897
12898	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12899	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12900	return VINF_SUCCESS;
12901	}
12902
12903
12904	/**
12905	* VM-exit exception handler for \#DB (Debug exception).
12906	*/
12907	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12908	{
12909	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12911	Log6(("XcptDB\n"));
12912
12913	/*
12914	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12915	* for processing.
12916	*/
12917	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12918	AssertRCReturn(rc, rc);
12919
12920	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12921	uint64_t uDR6 = X86_DR6_INIT_VAL;
12922	uDR6 \|= ( pVmxTransient->uExitQualification
12923	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
12924
12925	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12926	if (rc == VINF_EM_RAW_GUEST_TRAP)
12927	{
12928	/*
12929	* The exception was for the guest. Update DR6, DR7.GD and
12930	* IA32_DEBUGCTL.LBR before forwarding it.
12931	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12932	*/
12933	VMMRZCallRing3Disable(pVCpu);
12934	HM_DISABLE_PREEMPT();
12935
12936	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12937	pMixedCtx->dr[6] \|= uDR6;
12938	if (CPUMIsGuestDebugStateActive(pVCpu))
12939	ASMSetDR6(pMixedCtx->dr[6]);
12940
12941	HM_RESTORE_PREEMPT();
12942	VMMRZCallRing3Enable(pVCpu);
12943
12944	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12945	AssertRCReturn(rc, rc);
12946
12947	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12948	pMixedCtx->dr[7] &= ~X86_DR7_GD;
12949
12950	/* Paranoia. */
12951	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12952	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
12953
12954	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12955	AssertRCReturn(rc, rc);
12956
12957	/*
12958	* Raise #DB in the guest.
12959	*
12960	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
12961	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
12962	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
12963	*
12964	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
12965	*/
12966	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12967	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12968	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12969	AssertRCReturn(rc, rc);
12970	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12971	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12972	return VINF_SUCCESS;
12973	}
12974
12975	/*
12976	* Not a guest trap, must be a hypervisor related debug event then.
12977	* Update DR6 in case someone is interested in it.
12978	*/
12979	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12980	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12981	CPUMSetHyperDR6(pVCpu, uDR6);
12982
12983	return rc;
12984	}
12985
12986
12987	/**
12988	* VM-exit exception handler for \#NM (Device-not-available exception: floating
12989	* point exception).
12990	*/
12991	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12992	{
12993	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12994
12995	/* We require CR0 and EFER. EFER is always up-to-date. */
12996	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12997	AssertRCReturn(rc, rc);
12998
12999	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13000	VMMRZCallRing3Disable(pVCpu);
13001	HM_DISABLE_PREEMPT();
13002
13003	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13004	if (pVmxTransient->fWasGuestFPUStateActive)
13005	{
13006	rc = VINF_EM_RAW_GUEST_TRAP;
13007	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13008	}
13009	else
13010	{
13011	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13012	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13013	#endif
13014	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13015	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13016	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13017	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13018	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13019	}
13020
13021	HM_RESTORE_PREEMPT();
13022	VMMRZCallRing3Enable(pVCpu);
13023
13024	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13025	{
13026	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13027	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13028	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13029	pVCpu->hm.s.fPreloadGuestFpu = true;
13030	}
13031	else
13032	{
13033	/* Forward #NM to the guest. */
13034	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13035	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13036	AssertRCReturn(rc, rc);
13037	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13038	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13039	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13040	}
13041
13042	return VINF_SUCCESS;
13043	}
13044
13045
13046	/**
13047	* VM-exit exception handler for \#GP (General-protection exception).
13048	*
13049	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13050	*/
13051	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13052	{
13053	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13054	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13055
13056	int rc;
13057	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13058	{ /* likely */ }
13059	else
13060	{
13061	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13062	Assert(pVCpu->hm.s.fUsingDebugLoop);
13063	#endif
13064	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13065	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13066	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13067	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13068	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13069	AssertRCReturn(rc, rc);
13070	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13071	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13072	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13073	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13074	return rc;
13075	}
13076
13077	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13078	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13079
13080	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13081	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13082	AssertRCReturn(rc, rc);
13083
13084	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13085	uint32_t cbOp = 0;
13086	PVM pVM = pVCpu->CTX_SUFF(pVM);
13087	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13088	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13089	if (RT_SUCCESS(rc))
13090	{
13091	rc = VINF_SUCCESS;
13092	Assert(cbOp == pDis->cbInstr);
13093	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13094	switch (pDis->pCurInstr->uOpcode)
13095	{
13096	case OP_CLI:
13097	{
13098	pMixedCtx->eflags.Bits.u1IF = 0;
13099	pMixedCtx->eflags.Bits.u1RF = 0;
13100	pMixedCtx->rip += pDis->cbInstr;
13101	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13102	if ( !fDbgStepping
13103	&& pMixedCtx->eflags.Bits.u1TF)
13104	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13105	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13106	break;
13107	}
13108
13109	case OP_STI:
13110	{
13111	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13112	pMixedCtx->eflags.Bits.u1IF = 1;
13113	pMixedCtx->eflags.Bits.u1RF = 0;
13114	pMixedCtx->rip += pDis->cbInstr;
13115	if (!fOldIF)
13116	{
13117	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13118	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13119	}
13120	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13121	if ( !fDbgStepping
13122	&& pMixedCtx->eflags.Bits.u1TF)
13123	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13125	break;
13126	}
13127
13128	case OP_HLT:
13129	{
13130	rc = VINF_EM_HALT;
13131	pMixedCtx->rip += pDis->cbInstr;
13132	pMixedCtx->eflags.Bits.u1RF = 0;
13133	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13134	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13135	break;
13136	}
13137
13138	case OP_POPF:
13139	{
13140	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13141	uint32_t cbParm;
13142	uint32_t uMask;
13143	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13144	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13145	{
13146	cbParm = 4;
13147	uMask = 0xffffffff;
13148	}
13149	else
13150	{
13151	cbParm = 2;
13152	uMask = 0xffff;
13153	}
13154
13155	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13156	RTGCPTR GCPtrStack = 0;
13157	X86EFLAGS Eflags;
13158	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13159	&GCPtrStack);
13160	if (RT_SUCCESS(rc))
13161	{
13162	Assert(sizeof(Eflags.u32) >= cbParm);
13163	Eflags.u32 = 0;
13164	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13165	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13166	}
13167	if (RT_FAILURE(rc))
13168	{
13169	rc = VERR_EM_INTERPRETER;
13170	break;
13171	}
13172	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13173	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13174	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13175	pMixedCtx->esp += cbParm;
13176	pMixedCtx->esp &= uMask;
13177	pMixedCtx->rip += pDis->cbInstr;
13178	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13179	\| HM_CHANGED_GUEST_RSP
13180	\| HM_CHANGED_GUEST_RFLAGS);
13181	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13182	POPF restores EFLAGS.TF. */
13183	if ( !fDbgStepping
13184	&& fGstStepping)
13185	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13187	break;
13188	}
13189
13190	case OP_PUSHF:
13191	{
13192	uint32_t cbParm;
13193	uint32_t uMask;
13194	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13195	{
13196	cbParm = 4;
13197	uMask = 0xffffffff;
13198	}
13199	else
13200	{
13201	cbParm = 2;
13202	uMask = 0xffff;
13203	}
13204
13205	/* Get the stack pointer & push the contents of eflags onto the stack. */
13206	RTGCPTR GCPtrStack = 0;
13207	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13208	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13209	if (RT_FAILURE(rc))
13210	{
13211	rc = VERR_EM_INTERPRETER;
13212	break;
13213	}
13214	X86EFLAGS Eflags = pMixedCtx->eflags;
13215	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13216	Eflags.Bits.u1RF = 0;
13217	Eflags.Bits.u1VM = 0;
13218
13219	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13220	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13221	{
13222	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13223	rc = VERR_EM_INTERPRETER;
13224	break;
13225	}
13226	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13227	pMixedCtx->esp -= cbParm;
13228	pMixedCtx->esp &= uMask;
13229	pMixedCtx->rip += pDis->cbInstr;
13230	pMixedCtx->eflags.Bits.u1RF = 0;
13231	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13232	\| HM_CHANGED_GUEST_RSP
13233	\| HM_CHANGED_GUEST_RFLAGS);
13234	if ( !fDbgStepping
13235	&& pMixedCtx->eflags.Bits.u1TF)
13236	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13238	break;
13239	}
13240
13241	case OP_IRET:
13242	{
13243	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13244	* instruction reference. */
13245	RTGCPTR GCPtrStack = 0;
13246	uint32_t uMask = 0xffff;
13247	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13248	uint16_t aIretFrame[3];
13249	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13250	{
13251	rc = VERR_EM_INTERPRETER;
13252	break;
13253	}
13254	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13255	&GCPtrStack);
13256	if (RT_SUCCESS(rc))
13257	{
13258	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13259	PGMACCESSORIGIN_HM));
13260	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13261	}
13262	if (RT_FAILURE(rc))
13263	{
13264	rc = VERR_EM_INTERPRETER;
13265	break;
13266	}
13267	pMixedCtx->eip = 0;
13268	pMixedCtx->ip = aIretFrame[0];
13269	pMixedCtx->cs.Sel = aIretFrame[1];
13270	pMixedCtx->cs.ValidSel = aIretFrame[1];
13271	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13272	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13273	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13274	pMixedCtx->sp += sizeof(aIretFrame);
13275	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13276	\| HM_CHANGED_GUEST_SEGMENT_REGS
13277	\| HM_CHANGED_GUEST_RSP
13278	\| HM_CHANGED_GUEST_RFLAGS);
13279	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13280	if ( !fDbgStepping
13281	&& fGstStepping)
13282	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13283	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13284	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13285	break;
13286	}
13287
13288	case OP_INT:
13289	{
13290	uint16_t uVector = pDis->Param1.uValue & 0xff;
13291	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13292	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13293	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13294	break;
13295	}
13296
13297	case OP_INTO:
13298	{
13299	if (pMixedCtx->eflags.Bits.u1OF)
13300	{
13301	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13302	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13303	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13304	}
13305	else
13306	{
13307	pMixedCtx->eflags.Bits.u1RF = 0;
13308	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13309	}
13310	break;
13311	}
13312
13313	default:
13314	{
13315	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13316	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13317	EMCODETYPE_SUPERVISOR);
13318	rc = VBOXSTRICTRC_VAL(rc2);
13319	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13320	/** @todo We have to set pending-debug exceptions here when the guest is
13321	* single-stepping depending on the instruction that was interpreted. */
13322	Log4(("#GP rc=%Rrc\n", rc));
13323	break;
13324	}
13325	}
13326	}
13327	else
13328	rc = VERR_EM_INTERPRETER;
13329
13330	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13331	("#GP Unexpected rc=%Rrc\n", rc));
13332	return rc;
13333	}
13334
13335
13336	/**
13337	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13338	* the exception reported in the VMX transient structure back into the VM.
13339	*
13340	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13341	* up-to-date.
13342	*/
13343	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13344	{
13345	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13346	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13347	Assert(pVCpu->hm.s.fUsingDebugLoop);
13348	#endif
13349
13350	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13351	hmR0VmxCheckExitDueToEventDelivery(). */
13352	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13353	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13354	AssertRCReturn(rc, rc);
13355	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13356
13357	#ifdef DEBUG_ramshankar
13358	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13359	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13360	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13361	#endif
13362
13363	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13364	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13365	return VINF_SUCCESS;
13366	}
13367
13368
13369	/**
13370	* VM-exit exception handler for \#PF (Page-fault exception).
13371	*/
13372	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13373	{
13374	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13375	PVM pVM = pVCpu->CTX_SUFF(pVM);
13376	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13377	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13378	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13379	AssertRCReturn(rc, rc);
13380
13381	if (!pVM->hm.s.fNestedPaging)
13382	{ /* likely */ }
13383	else
13384	{
13385	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13386	Assert(pVCpu->hm.s.fUsingDebugLoop);
13387	#endif
13388	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13389	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13390	{
13391	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13392	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13393	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13394	}
13395	else
13396	{
13397	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13398	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13399	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13400	}
13401	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13402	return rc;
13403	}
13404
13405	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13406	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13407	if (pVmxTransient->fVectoringPF)
13408	{
13409	Assert(pVCpu->hm.s.Event.fPending);
13410	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13411	}
13412
13413	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13414	AssertRCReturn(rc, rc);
13415
13416	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13417	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13418
13419	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13420	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13421	(RTGCPTR)pVmxTransient->uExitQualification);
13422
13423	Log4(("#PF: rc=%Rrc\n", rc));
13424	if (rc == VINF_SUCCESS)
13425	{
13426	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13427	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13428	* memory? We don't update the whole state here... */
13429	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13430	\| HM_CHANGED_GUEST_RSP
13431	\| HM_CHANGED_GUEST_RFLAGS
13432	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13433	TRPMResetTrap(pVCpu);
13434	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13435	return rc;
13436	}
13437
13438	if (rc == VINF_EM_RAW_GUEST_TRAP)
13439	{
13440	if (!pVmxTransient->fVectoringDoublePF)
13441	{
13442	/* It's a guest page fault and needs to be reflected to the guest. */
13443	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13444	TRPMResetTrap(pVCpu);
13445	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13446	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13447	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13448	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13449	}
13450	else
13451	{
13452	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13453	TRPMResetTrap(pVCpu);
13454	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13455	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13456	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13457	}
13458
13459	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13460	return VINF_SUCCESS;
13461	}
13462
13463	TRPMResetTrap(pVCpu);
13464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13465	return rc;
13466	}
13467
13468	/** @} */
13469

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 61605

Download in other formats: