HMVMXR0.cpp@ 72643

Last change on this file since 72643 was 72643, checked in by vboxsync, 6 years ago
VMM: Make SVM R0 code use CPUMCTX_EXTRN_xxx flags and cleanups. bugref:9193
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 621.3 KB

Line
1	/* $Id: HMVMXR0.cpp 72643 2018-06-21 16:02:03Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/gim.h>
35	#include <VBox/vmm/apic.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#include "HMInternal.h"
40	#include <VBox/vmm/vm.h>
41	#include "HMVMXR0.h"
42	#include "dtrace/VBoxVMM.h"
43
44	#define HMVMX_USE_IEM_EVENT_REFLECTION
45	#ifdef DEBUG_ramshankar
46	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51	# define HMVMX_ALWAYS_TRAP_PF
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
140	* guest using hardware-assisted VMX.
141	*
142	* This excludes state like GPRs (other than RSP) which are always are
143	* swapped and restored across the world-switch and also registers like EFER,
144	* MSR which cannot be modified by the guest without causing a VM-exit.
145	*/
146	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
147	\| CPUMCTX_EXTRN_RFLAGS \
148	\| CPUMCTX_EXTRN_SREG_MASK \
149	\| CPUMCTX_EXTRN_TABLE_MASK \
150	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
151	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
152	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
153	\| CPUMCTX_EXTRN_TSC_AUX \
154	\| CPUMCTX_EXTRN_OTHER_MSRS \
155	\| CPUMCTX_EXTRN_CR0 \
156	\| CPUMCTX_EXTRN_CR3 \
157	\| CPUMCTX_EXTRN_CR4 \
158	\| CPUMCTX_EXTRN_DR7)
159
160	/**
161	* Exception bitmap mask for real-mode guests (real-on-v86).
162	*
163	* We need to intercept all exceptions manually except:
164	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
165	* due to bugs in Intel CPUs.
166	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
167	* support.
168	*/
169	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
170	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
171	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
172	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
173	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
174	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
175	\| RT_BIT(X86_XCPT_XF))
176
177	/**
178	* Exception bitmap mask for all contributory exceptions.
179	*
180	* Page fault is deliberately excluded here as it's conditional as to whether
181	* it's contributory or benign. Page faults are handled separately.
182	*/
183	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
184	\| RT_BIT(X86_XCPT_DE))
185
186	/** Maximum VM-instruction error number. */
187	#define HMVMX_INSTR_ERROR_MAX 28
188
189	/** Profiling macro. */
190	#ifdef HM_PROFILE_EXIT_DISPATCH
191	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
192	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
193	#else
194	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
195	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
196	#endif
197
198	/** Assert that preemption is disabled or covered by thread-context hooks. */
199	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
200	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
201
202	/** Assert that we haven't migrated CPUs when thread-context hooks are not
203	* used. */
204	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
205	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
206	("Illegal migration! Entered on CPU %u Current %u\n", \
207	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
208
209	/** Helper macro for VM-exit handlers called unexpectedly. */
210	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
211	do { \
212	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
213	return VERR_VMX_UNEXPECTED_EXIT; \
214	} while (0)
215
216	/** Macro for saving segment registers from VMCS into the guest-CPU
217	* context. */
218	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
219	# define HMVMX_SAVE_SREG(Sel, a_pCtxSelReg) \
220	hmR0VmxSaveSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
221	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
222	#else
223	# define HMVMX_SAVE_SREG(Sel, a_pCtxSelReg) \
224	hmR0VmxSaveSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
225	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
226	#endif
227
228
229	/*********************************************************************************************************************************
230	* Structures and Typedefs *
231	*********************************************************************************************************************************/
232	/**
233	* VMX transient state.
234	*
235	* A state structure for holding miscellaneous information across
236	* VMX non-root operation and restored after the transition.
237	*/
238	typedef struct VMXTRANSIENT
239	{
240	/** The host's rflags/eflags. */
241	RTCCUINTREG fEFlags;
242	#if HC_ARCH_BITS == 32
243	uint32_t u32Alignment0;
244	#endif
245	/** The guest's TPR value used for TPR shadowing. */
246	uint8_t u8GuestTpr;
247	/** Alignment. */
248	uint8_t abAlignment0[7];
249
250	/** The basic VM-exit reason. */
251	uint16_t uExitReason;
252	/** Alignment. */
253	uint16_t u16Alignment0;
254	/** The VM-exit interruption error code. */
255	uint32_t uExitIntErrorCode;
256	/** The VM-exit exit code qualification. */
257	uint64_t uExitQualification;
258
259	/** The VM-exit interruption-information field. */
260	uint32_t uExitIntInfo;
261	/** The VM-exit instruction-length field. */
262	uint32_t cbInstr;
263	/** The VM-exit instruction-information field. */
264	union
265	{
266	/** Plain unsigned int representation. */
267	uint32_t u;
268	/** INS and OUTS information. */
269	struct
270	{
271	uint32_t u7Reserved0 : 7;
272	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
273	uint32_t u3AddrSize : 3;
274	uint32_t u5Reserved1 : 5;
275	/** The segment register (X86_SREG_XXX). */
276	uint32_t iSegReg : 3;
277	uint32_t uReserved2 : 14;
278	} StrIo;
279	/** INVEPT, INVVPID, INVPCID information. */
280	struct
281	{
282	/** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
283	uint32_t u2Scaling : 2;
284	uint32_t u5Reserved0 : 5;
285	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
286	uint32_t u3AddrSize : 3;
287	uint32_t u1Reserved0 : 1;
288	uint32_t u4Reserved0 : 4;
289	/** The segment register (X86_SREG_XXX). */
290	uint32_t iSegReg : 3;
291	/** The index register (X86_GREG_XXX). */
292	uint32_t iIdxReg : 4;
293	/** Set if index register is invalid. */
294	uint32_t fIdxRegValid : 1;
295	/** The base register (X86_GREG_XXX). */
296	uint32_t iBaseReg : 4;
297	/** Set if base register is invalid. */
298	uint32_t fBaseRegValid : 1;
299	/** Register 2 (X86_GREG_XXX). */
300	uint32_t iReg2 : 4;
301	} Inv;
302	} ExitInstrInfo;
303	/** Whether the VM-entry failed or not. */
304	bool fVMEntryFailed;
305	/** Alignment. */
306	uint8_t abAlignment1[3];
307
308	/** The VM-entry interruption-information field. */
309	uint32_t uEntryIntInfo;
310	/** The VM-entry exception error code field. */
311	uint32_t uEntryXcptErrorCode;
312	/** The VM-entry instruction length field. */
313	uint32_t cbEntryInstr;
314
315	/** IDT-vectoring information field. */
316	uint32_t uIdtVectoringInfo;
317	/** IDT-vectoring error code. */
318	uint32_t uIdtVectoringErrorCode;
319
320	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
321	uint32_t fVmcsFieldsRead;
322
323	/** Whether the guest debug state was active at the time of VM-exit. */
324	bool fWasGuestDebugStateActive;
325	/** Whether the hyper debug state was active at the time of VM-exit. */
326	bool fWasHyperDebugStateActive;
327	/** Whether TSC-offsetting should be setup before VM-entry. */
328	bool fUpdateTscOffsettingAndPreemptTimer;
329	/** Whether the VM-exit was caused by a page-fault during delivery of a
330	* contributory exception or a page-fault. */
331	bool fVectoringDoublePF;
332	/** Whether the VM-exit was caused by a page-fault during delivery of an
333	* external interrupt or NMI. */
334	bool fVectoringPF;
335	} VMXTRANSIENT;
336	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
337	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
338	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
339	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
340	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
341	/** Pointer to VMX transient state. */
342	typedef VMXTRANSIENT *PVMXTRANSIENT;
343
344
345	/**
346	* MSR-bitmap read permissions.
347	*/
348	typedef enum VMXMSREXITREAD
349	{
350	/** Reading this MSR causes a VM-exit. */
351	VMXMSREXIT_INTERCEPT_READ = 0xb,
352	/** Reading this MSR does not cause a VM-exit. */
353	VMXMSREXIT_PASSTHRU_READ
354	} VMXMSREXITREAD;
355	/** Pointer to MSR-bitmap read permissions. */
356	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
357
358	/**
359	* MSR-bitmap write permissions.
360	*/
361	typedef enum VMXMSREXITWRITE
362	{
363	/** Writing to this MSR causes a VM-exit. */
364	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
365	/** Writing to this MSR does not cause a VM-exit. */
366	VMXMSREXIT_PASSTHRU_WRITE
367	} VMXMSREXITWRITE;
368	/** Pointer to MSR-bitmap write permissions. */
369	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
370
371
372	/**
373	* VMX VM-exit handler.
374	*
375	* @returns Strict VBox status code (i.e. informational status codes too).
376	* @param pVCpu The cross context virtual CPU structure.
377	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
378	* out-of-sync. Make sure to update the required
379	* fields before using them.
380	* @param pVmxTransient Pointer to the VMX-transient structure.
381	*/
382	#ifndef HMVMX_USE_FUNCTION_TABLE
383	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384	#else
385	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386	/** Pointer to VM-exit handler. */
387	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
388	#endif
389
390	/**
391	* VMX VM-exit handler, non-strict status code.
392	*
393	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
394	*
395	* @returns VBox status code, no informational status code returned.
396	* @param pVCpu The cross context virtual CPU structure.
397	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
398	* out-of-sync. Make sure to update the required
399	* fields before using them.
400	* @param pVmxTransient Pointer to the VMX-transient structure.
401	*
402	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
403	* use of that status code will be replaced with VINF_EM_SOMETHING
404	* later when switching over to IEM.
405	*/
406	#ifndef HMVMX_USE_FUNCTION_TABLE
407	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#else
409	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
410	#endif
411
412
413	/*********************************************************************************************************************************
414	* Internal Functions *
415	*********************************************************************************************************************************/
416	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
417	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
418	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
419	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
420	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
421	bool fStepping, uint32_t *puIntState);
422	#if HC_ARCH_BITS == 32
423	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
424	#endif
425	#ifndef HMVMX_USE_FUNCTION_TABLE
426	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
427	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
428	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
429	#else
430	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
431	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
432	#endif
433
434
435	/** @name VM-exit handlers.
436	* @{
437	*/
438	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
439	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
440	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
441	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
442	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
443	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
444	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
445	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
446	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
447	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
448	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
449	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
450	static FNVMXEXITHANDLER hmR0VmxExitHlt;
451	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
452	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
453	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
454	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
455	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
456	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
457	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
458	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
459	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
460	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
461	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
462	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
463	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
464	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
465	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
466	static FNVMXEXITHANDLER hmR0VmxExitMwait;
467	static FNVMXEXITHANDLER hmR0VmxExitMtf;
468	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
469	static FNVMXEXITHANDLER hmR0VmxExitPause;
470	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
471	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
472	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
473	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
474	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
475	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
476	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
477	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
478	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
479	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
480	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
481	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
482	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
483	/** @} */
484
485	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
486	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
487	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
488	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
489	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
490	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
491	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
492	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
493
494
495	/*********************************************************************************************************************************
496	* Global Variables *
497	*********************************************************************************************************************************/
498	#ifdef HMVMX_USE_FUNCTION_TABLE
499
500	/**
501	* VMX_EXIT dispatch table.
502	*/
503	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
504	{
505	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
506	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
507	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
508	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
509	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
510	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
511	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
512	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
513	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
514	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
515	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
516	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
517	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
518	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
519	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
520	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
521	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
522	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
523	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
524	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
525	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
526	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
527	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
528	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
529	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
530	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
531	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
532	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#ifdef VBOX_STRICT
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609
610	/**
611	* Updates the VM's last error record.
612	*
613	* If there was a VMX instruction error, reads the error data from the VMCS and
614	* updates VCPU's last error record as well.
615	*
616	* @param pVM The cross context VM structure.
617	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
618	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
619	* VERR_VMX_INVALID_VMCS_FIELD.
620	* @param rc The error code.
621	*/
622	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
623	{
624	AssertPtr(pVM);
625	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
626	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
627	{
628	AssertPtrReturnVoid(pVCpu);
629	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
630	}
631	pVM->hm.s.lLastError = rc;
632	}
633
634
635	/**
636	* Reads the VM-entry interruption-information field from the VMCS into the VMX
637	* transient structure.
638	*
639	* @returns VBox status code.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*
642	* @remarks No-long-jump zone!!!
643	*/
644	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
647	AssertRCReturn(rc, rc);
648	return VINF_SUCCESS;
649	}
650
651
652	#ifdef VBOX_STRICT
653	/**
654	* Reads the VM-entry exception error code field from the VMCS into
655	* the VMX transient structure.
656	*
657	* @returns VBox status code.
658	* @param pVmxTransient Pointer to the VMX transient structure.
659	*
660	* @remarks No-long-jump zone!!!
661	*/
662	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
665	AssertRCReturn(rc, rc);
666	return VINF_SUCCESS;
667	}
668	#endif /* VBOX_STRICT */
669
670
671	#ifdef VBOX_STRICT
672	/**
673	* Reads the VM-entry exception error code field from the VMCS into
674	* the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVmxTransient Pointer to the VMX transient structure.
678	*
679	* @remarks No-long-jump zone!!!
680	*/
681	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682	{
683	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
684	AssertRCReturn(rc, rc);
685	return VINF_SUCCESS;
686	}
687	#endif /* VBOX_STRICT */
688
689
690	/**
691	* Reads the VM-exit interruption-information field from the VMCS into the VMX
692	* transient structure.
693	*
694	* @returns VBox status code.
695	* @param pVmxTransient Pointer to the VMX transient structure.
696	*/
697	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
698	{
699	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
700	{
701	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
702	AssertRCReturn(rc, rc);
703	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
704	}
705	return VINF_SUCCESS;
706	}
707
708
709	/**
710	* Reads the VM-exit interruption error code from the VMCS into the VMX
711	* transient structure.
712	*
713	* @returns VBox status code.
714	* @param pVmxTransient Pointer to the VMX transient structure.
715	*/
716	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
717	{
718	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
719	{
720	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
721	AssertRCReturn(rc, rc);
722	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
723	}
724	return VINF_SUCCESS;
725	}
726
727
728	/**
729	* Reads the VM-exit instruction length field from the VMCS into the VMX
730	* transient structure.
731	*
732	* @returns VBox status code.
733	* @param pVmxTransient Pointer to the VMX transient structure.
734	*/
735	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
736	{
737	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
738	{
739	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
740	AssertRCReturn(rc, rc);
741	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
742	}
743	return VINF_SUCCESS;
744	}
745
746
747	/**
748	* Reads the VM-exit instruction-information field from the VMCS into
749	* the VMX transient structure.
750	*
751	* @returns VBox status code.
752	* @param pVmxTransient Pointer to the VMX transient structure.
753	*/
754	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
755	{
756	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
757	{
758	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
759	AssertRCReturn(rc, rc);
760	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
761	}
762	return VINF_SUCCESS;
763	}
764
765
766	/**
767	* Reads the exit code qualification from the VMCS into the VMX transient
768	* structure.
769	*
770	* @returns VBox status code.
771	* @param pVCpu The cross context virtual CPU structure of the
772	* calling EMT. (Required for the VMCS cache case.)
773	* @param pVmxTransient Pointer to the VMX transient structure.
774	*/
775	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
776	{
777	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
778	{
779	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
780	AssertRCReturn(rc, rc);
781	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
782	}
783	return VINF_SUCCESS;
784	}
785
786
787	/**
788	* Reads the IDT-vectoring information field from the VMCS into the VMX
789	* transient structure.
790	*
791	* @returns VBox status code.
792	* @param pVmxTransient Pointer to the VMX transient structure.
793	*
794	* @remarks No-long-jump zone!!!
795	*/
796	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
797	{
798	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
799	{
800	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
801	AssertRCReturn(rc, rc);
802	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
803	}
804	return VINF_SUCCESS;
805	}
806
807
808	/**
809	* Reads the IDT-vectoring error code from the VMCS into the VMX
810	* transient structure.
811	*
812	* @returns VBox status code.
813	* @param pVmxTransient Pointer to the VMX transient structure.
814	*/
815	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
816	{
817	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
818	{
819	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
820	AssertRCReturn(rc, rc);
821	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
822	}
823	return VINF_SUCCESS;
824	}
825
826
827	/**
828	* Enters VMX root mode operation on the current CPU.
829	*
830	* @returns VBox status code.
831	* @param pVM The cross context VM structure. Can be
832	* NULL, after a resume.
833	* @param HCPhysCpuPage Physical address of the VMXON region.
834	* @param pvCpuPage Pointer to the VMXON region.
835	*/
836	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
837	{
838	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
839	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
840	Assert(pvCpuPage);
841	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
842
843	if (pVM)
844	{
845	/* Write the VMCS revision dword to the VMXON region. */
846	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
847	}
848
849	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
850	RTCCUINTREG fEFlags = ASMIntDisableFlags();
851
852	/* Enable the VMX bit in CR4 if necessary. */
853	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
854
855	/* Enter VMX root mode. */
856	int rc = VMXEnable(HCPhysCpuPage);
857	if (RT_FAILURE(rc))
858	{
859	if (!(uOldCr4 & X86_CR4_VMXE))
860	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
861
862	if (pVM)
863	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
864	}
865
866	/* Restore interrupts. */
867	ASMSetFlags(fEFlags);
868	return rc;
869	}
870
871
872	/**
873	* Exits VMX root mode operation on the current CPU.
874	*
875	* @returns VBox status code.
876	*/
877	static int hmR0VmxLeaveRootMode(void)
878	{
879	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
880
881	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
882	RTCCUINTREG fEFlags = ASMIntDisableFlags();
883
884	/* If we're for some reason not in VMX root mode, then don't leave it. */
885	RTCCUINTREG uHostCR4 = ASMGetCR4();
886
887	int rc;
888	if (uHostCR4 & X86_CR4_VMXE)
889	{
890	/* Exit VMX root mode and clear the VMX bit in CR4. */
891	VMXDisable();
892	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
893	rc = VINF_SUCCESS;
894	}
895	else
896	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
897
898	/* Restore interrupts. */
899	ASMSetFlags(fEFlags);
900	return rc;
901	}
902
903
904	/**
905	* Allocates and maps one physically contiguous page. The allocated page is
906	* zero'd out. (Used by various VT-x structures).
907	*
908	* @returns IPRT status code.
909	* @param pMemObj Pointer to the ring-0 memory object.
910	* @param ppVirt Where to store the virtual address of the
911	* allocation.
912	* @param pHCPhys Where to store the physical address of the
913	* allocation.
914	*/
915	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
916	{
917	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
918	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
919	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
920
921	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
922	if (RT_FAILURE(rc))
923	return rc;
924	ppVirt = RTR0MemObjAddress(pMemObj);
925	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
926	ASMMemZero32(*ppVirt, PAGE_SIZE);
927	return VINF_SUCCESS;
928	}
929
930
931	/**
932	* Frees and unmaps an allocated physical page.
933	*
934	* @param pMemObj Pointer to the ring-0 memory object.
935	* @param ppVirt Where to re-initialize the virtual address of
936	* allocation as 0.
937	* @param pHCPhys Where to re-initialize the physical address of the
938	* allocation as 0.
939	*/
940	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
941	{
942	AssertPtr(pMemObj);
943	AssertPtr(ppVirt);
944	AssertPtr(pHCPhys);
945	if (*pMemObj != NIL_RTR0MEMOBJ)
946	{
947	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
948	AssertRC(rc);
949	*pMemObj = NIL_RTR0MEMOBJ;
950	*ppVirt = 0;
951	*pHCPhys = 0;
952	}
953	}
954
955
956	/**
957	* Worker function to free VT-x related structures.
958	*
959	* @returns IPRT status code.
960	* @param pVM The cross context VM structure.
961	*/
962	static void hmR0VmxStructsFree(PVM pVM)
963	{
964	for (VMCPUID i = 0; i < pVM->cCpus; i++)
965	{
966	PVMCPU pVCpu = &pVM->aCpus[i];
967	AssertPtr(pVCpu);
968
969	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
970	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
971
972	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
973	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
974
975	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
976	}
977
978	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
979	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
980	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
981	#endif
982	}
983
984
985	/**
986	* Worker function to allocate VT-x related VM structures.
987	*
988	* @returns IPRT status code.
989	* @param pVM The cross context VM structure.
990	*/
991	static int hmR0VmxStructsAlloc(PVM pVM)
992	{
993	/*
994	* Initialize members up-front so we can cleanup properly on allocation failure.
995	*/
996	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
997	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
998	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
999	pVM->hm.s.vmx.HCPhys##a_Name = 0;
1000
1001	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
1002	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
1003	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
1004	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
1005
1006	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1007	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
1008	#endif
1009	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
1010
1011	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
1012	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1013	{
1014	PVMCPU pVCpu = &pVM->aCpus[i];
1015	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
1016	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
1017	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
1018	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
1019	}
1020	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
1021	#undef VMXLOCAL_INIT_VM_MEMOBJ
1022
1023	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
1024	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
1025	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
1026	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
1027
1028	/*
1029	* Allocate all the VT-x structures.
1030	*/
1031	int rc = VINF_SUCCESS;
1032	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1033	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1034	if (RT_FAILURE(rc))
1035	goto cleanup;
1036	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1037	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1038	#endif
1039
1040	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1041	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1042	{
1043	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1044	&pVM->hm.s.vmx.HCPhysApicAccess);
1045	if (RT_FAILURE(rc))
1046	goto cleanup;
1047	}
1048
1049	/*
1050	* Initialize per-VCPU VT-x structures.
1051	*/
1052	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1053	{
1054	PVMCPU pVCpu = &pVM->aCpus[i];
1055	AssertPtr(pVCpu);
1056
1057	/* Allocate the VM control structure (VMCS). */
1058	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1059	if (RT_FAILURE(rc))
1060	goto cleanup;
1061
1062	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1063	if ( PDMHasApic(pVM)
1064	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1065	{
1066	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1067	NULL /* pR3Ptr /, NULL / pRCPtr */);
1068	if (RT_FAILURE(rc))
1069	goto cleanup;
1070	}
1071
1072	/*
1073	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1074	* transparent accesses of specific MSRs.
1075	*
1076	* If the condition for enabling MSR bitmaps changes here, don't forget to
1077	* update HMAreMsrBitmapsAvailable().
1078	*/
1079	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1080	{
1081	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1082	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1083	if (RT_FAILURE(rc))
1084	goto cleanup;
1085	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1086	}
1087
1088	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1089	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1090	if (RT_FAILURE(rc))
1091	goto cleanup;
1092
1093	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1094	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1095	if (RT_FAILURE(rc))
1096	goto cleanup;
1097	}
1098
1099	return VINF_SUCCESS;
1100
1101	cleanup:
1102	hmR0VmxStructsFree(pVM);
1103	return rc;
1104	}
1105
1106
1107	/**
1108	* Does global VT-x initialization (called during module initialization).
1109	*
1110	* @returns VBox status code.
1111	*/
1112	VMMR0DECL(int) VMXR0GlobalInit(void)
1113	{
1114	#ifdef HMVMX_USE_FUNCTION_TABLE
1115	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1116	# ifdef VBOX_STRICT
1117	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1118	Assert(g_apfnVMExitHandlers[i]);
1119	# endif
1120	#endif
1121	return VINF_SUCCESS;
1122	}
1123
1124
1125	/**
1126	* Does global VT-x termination (called during module termination).
1127	*/
1128	VMMR0DECL(void) VMXR0GlobalTerm()
1129	{
1130	/* Nothing to do currently. */
1131	}
1132
1133
1134	/**
1135	* Sets up and activates VT-x on the current CPU.
1136	*
1137	* @returns VBox status code.
1138	* @param pCpu Pointer to the global CPU info struct.
1139	* @param pVM The cross context VM structure. Can be
1140	* NULL after a host resume operation.
1141	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1142	* fEnabledByHost is @c true).
1143	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1144	* @a fEnabledByHost is @c true).
1145	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1146	* enable VT-x on the host.
1147	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1148	*/
1149	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1150	void *pvMsrs)
1151	{
1152	Assert(pCpu);
1153	Assert(pvMsrs);
1154	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1155
1156	/* Enable VT-x if it's not already enabled by the host. */
1157	if (!fEnabledByHost)
1158	{
1159	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1160	if (RT_FAILURE(rc))
1161	return rc;
1162	}
1163
1164	/*
1165	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1166	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1167	*/
1168	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1169	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1170	{
1171	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1172	pCpu->fFlushAsidBeforeUse = false;
1173	}
1174	else
1175	pCpu->fFlushAsidBeforeUse = true;
1176
1177	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1178	++pCpu->cTlbFlushes;
1179
1180	return VINF_SUCCESS;
1181	}
1182
1183
1184	/**
1185	* Deactivates VT-x on the current CPU.
1186	*
1187	* @returns VBox status code.
1188	* @param pCpu Pointer to the global CPU info struct.
1189	* @param pvCpuPage Pointer to the VMXON region.
1190	* @param HCPhysCpuPage Physical address of the VMXON region.
1191	*
1192	* @remarks This function should never be called when SUPR0EnableVTx() or
1193	* similar was used to enable VT-x on the host.
1194	*/
1195	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1196	{
1197	NOREF(pCpu);
1198	NOREF(pvCpuPage);
1199	NOREF(HCPhysCpuPage);
1200
1201	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1202	return hmR0VmxLeaveRootMode();
1203	}
1204
1205
1206	/**
1207	* Sets the permission bits for the specified MSR in the MSR bitmap.
1208	*
1209	* @param pVCpu The cross context virtual CPU structure.
1210	* @param uMsr The MSR value.
1211	* @param enmRead Whether reading this MSR causes a VM-exit.
1212	* @param enmWrite Whether writing this MSR causes a VM-exit.
1213	*/
1214	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1215	{
1216	int32_t iBit;
1217	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1218
1219	/*
1220	* Layout:
1221	* 0x000 - 0x3ff - Low MSR read bits
1222	* 0x400 - 0x7ff - High MSR read bits
1223	* 0x800 - 0xbff - Low MSR write bits
1224	* 0xc00 - 0xfff - High MSR write bits
1225	*/
1226	if (uMsr <= 0x00001FFF)
1227	iBit = uMsr;
1228	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1229	{
1230	iBit = uMsr - UINT32_C(0xC0000000);
1231	pbMsrBitmap += 0x400;
1232	}
1233	else
1234	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1235
1236	Assert(iBit <= 0x1fff);
1237	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1238	ASMBitSet(pbMsrBitmap, iBit);
1239	else
1240	ASMBitClear(pbMsrBitmap, iBit);
1241
1242	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1243	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1244	else
1245	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1246	}
1247
1248
1249	#ifdef VBOX_STRICT
1250	/**
1251	* Gets the permission bits for the specified MSR in the MSR bitmap.
1252	*
1253	* @returns VBox status code.
1254	* @retval VINF_SUCCESS if the specified MSR is found.
1255	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1256	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1257	*
1258	* @param pVCpu The cross context virtual CPU structure.
1259	* @param uMsr The MSR.
1260	* @param penmRead Where to store the read permissions.
1261	* @param penmWrite Where to store the write permissions.
1262	*/
1263	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1264	{
1265	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1266	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1267	int32_t iBit;
1268	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1269
1270	/* See hmR0VmxSetMsrPermission() for the layout. */
1271	if (uMsr <= 0x00001FFF)
1272	iBit = uMsr;
1273	else if ( uMsr >= 0xC0000000
1274	&& uMsr <= 0xC0001FFF)
1275	{
1276	iBit = (uMsr - 0xC0000000);
1277	pbMsrBitmap += 0x400;
1278	}
1279	else
1280	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1281
1282	Assert(iBit <= 0x1fff);
1283	if (ASMBitTest(pbMsrBitmap, iBit))
1284	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1285	else
1286	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1287
1288	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1289	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1290	else
1291	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1292	return VINF_SUCCESS;
1293	}
1294	#endif /* VBOX_STRICT */
1295
1296
1297	/**
1298	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1299	* area.
1300	*
1301	* @returns VBox status code.
1302	* @param pVCpu The cross context virtual CPU structure.
1303	* @param cMsrs The number of MSRs.
1304	*/
1305	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1306	{
1307	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1308	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1309	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1310	{
1311	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1312	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1313	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1314	}
1315
1316	/* Update number of guest MSRs to load/store across the world-switch. */
1317	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1318	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1319
1320	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1321	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1322	AssertRCReturn(rc, rc);
1323
1324	/* Update the VCPU's copy of the MSR count. */
1325	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1326
1327	return VINF_SUCCESS;
1328	}
1329
1330
1331	/**
1332	* Adds a new (or updates the value of an existing) guest/host MSR
1333	* pair to be swapped during the world-switch as part of the
1334	* auto-load/store MSR area in the VMCS.
1335	*
1336	* @returns VBox status code.
1337	* @param pVCpu The cross context virtual CPU structure.
1338	* @param uMsr The MSR.
1339	* @param uGuestMsrValue Value of the guest MSR.
1340	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1341	* necessary.
1342	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1343	* its value was updated. Optional, can be NULL.
1344	*/
1345	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1346	bool *pfAddedAndUpdated)
1347	{
1348	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1349	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1350	uint32_t i;
1351	for (i = 0; i < cMsrs; i++)
1352	{
1353	if (pGuestMsr->u32Msr == uMsr)
1354	break;
1355	pGuestMsr++;
1356	}
1357
1358	bool fAdded = false;
1359	if (i == cMsrs)
1360	{
1361	++cMsrs;
1362	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1363	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1364
1365	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1366	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1367	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1368
1369	fAdded = true;
1370	}
1371
1372	/* Update the MSR values in the auto-load/store MSR area. */
1373	pGuestMsr->u32Msr = uMsr;
1374	pGuestMsr->u64Value = uGuestMsrValue;
1375
1376	/* Create/update the MSR slot in the host MSR area. */
1377	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378	pHostMsr += i;
1379	pHostMsr->u32Msr = uMsr;
1380
1381	/*
1382	* Update the host MSR only when requested by the caller AND when we're
1383	* adding it to the auto-load/store area. Otherwise, it would have been
1384	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1385	*/
1386	bool fUpdatedMsrValue = false;
1387	if ( fAdded
1388	&& fUpdateHostMsr)
1389	{
1390	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1391	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1392	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1393	fUpdatedMsrValue = true;
1394	}
1395
1396	if (pfAddedAndUpdated)
1397	*pfAddedAndUpdated = fUpdatedMsrValue;
1398	return VINF_SUCCESS;
1399	}
1400
1401
1402	/**
1403	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1404	* auto-load/store MSR area in the VMCS.
1405	*
1406	* @returns VBox status code.
1407	* @param pVCpu The cross context virtual CPU structure.
1408	* @param uMsr The MSR.
1409	*/
1410	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1411	{
1412	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1413	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1414	for (uint32_t i = 0; i < cMsrs; i++)
1415	{
1416	/* Find the MSR. */
1417	if (pGuestMsr->u32Msr == uMsr)
1418	{
1419	/* If it's the last MSR, simply reduce the count. */
1420	if (i == cMsrs - 1)
1421	{
1422	--cMsrs;
1423	break;
1424	}
1425
1426	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1427	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1428	pLastGuestMsr += cMsrs - 1;
1429	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1430	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1431
1432	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1433	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1434	pLastHostMsr += cMsrs - 1;
1435	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1436	pHostMsr->u64Value = pLastHostMsr->u64Value;
1437	--cMsrs;
1438	break;
1439	}
1440	pGuestMsr++;
1441	}
1442
1443	/* Update the VMCS if the count changed (meaning the MSR was found). */
1444	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1445	{
1446	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1447	AssertRCReturn(rc, rc);
1448
1449	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1450	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1451	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1452
1453	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1454	return VINF_SUCCESS;
1455	}
1456
1457	return VERR_NOT_FOUND;
1458	}
1459
1460
1461	/**
1462	* Checks if the specified guest MSR is part of the auto-load/store area in
1463	* the VMCS.
1464	*
1465	* @returns true if found, false otherwise.
1466	* @param pVCpu The cross context virtual CPU structure.
1467	* @param uMsr The MSR to find.
1468	*/
1469	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1470	{
1471	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1472	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1473
1474	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1475	{
1476	if (pGuestMsr->u32Msr == uMsr)
1477	return true;
1478	}
1479	return false;
1480	}
1481
1482
1483	/**
1484	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1485	*
1486	* @param pVCpu The cross context virtual CPU structure.
1487	*
1488	* @remarks No-long-jump zone!!!
1489	*/
1490	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1491	{
1492	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1493	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1494	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1495	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1496
1497	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1498	{
1499	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1500
1501	/*
1502	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1503	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1504	*/
1505	if (pHostMsr->u32Msr == MSR_K6_EFER)
1506	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1507	else
1508	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1509	}
1510
1511	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1512	}
1513
1514
1515	/**
1516	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1517	* perform lazy restoration of the host MSRs while leaving VT-x.
1518	*
1519	* @param pVCpu The cross context virtual CPU structure.
1520	*
1521	* @remarks No-long-jump zone!!!
1522	*/
1523	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1524	{
1525	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1526
1527	/*
1528	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1529	*/
1530	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1531	{
1532	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1533	#if HC_ARCH_BITS == 64
1534	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1535	{
1536	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1537	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1538	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1539	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1540	}
1541	#endif
1542	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1543	}
1544	}
1545
1546
1547	/**
1548	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1549	* lazily while leaving VT-x.
1550	*
1551	* @returns true if it does, false otherwise.
1552	* @param pVCpu The cross context virtual CPU structure.
1553	* @param uMsr The MSR to check.
1554	*/
1555	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1556	{
1557	NOREF(pVCpu);
1558	#if HC_ARCH_BITS == 64
1559	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1560	{
1561	switch (uMsr)
1562	{
1563	case MSR_K8_LSTAR:
1564	case MSR_K6_STAR:
1565	case MSR_K8_SF_MASK:
1566	case MSR_K8_KERNEL_GS_BASE:
1567	return true;
1568	}
1569	}
1570	#else
1571	RT_NOREF(pVCpu, uMsr);
1572	#endif
1573	return false;
1574	}
1575
1576
1577	/**
1578	* Saves a set of guest MSRs back into the guest-CPU context.
1579	*
1580	* @param pVCpu The cross context virtual CPU structure.
1581	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1582	* out-of-sync. Make sure to update the required fields
1583	* before using them.
1584	*
1585	* @remarks No-long-jump zone!!!
1586	*/
1587	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1588	{
1589	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1590	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1591
1592	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1593	{
1594	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1595	#if HC_ARCH_BITS == 64
1596	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1597	{
1598	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1599	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1600	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1601	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1602	}
1603	#else
1604	NOREF(pMixedCtx);
1605	#endif
1606	}
1607	}
1608
1609
1610	/**
1611	* Loads a set of guests MSRs to allow read/passthru to the guest.
1612	*
1613	* The name of this function is slightly confusing. This function does NOT
1614	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1615	* common prefix for functions dealing with "lazy restoration" of the shared
1616	* MSRs.
1617	*
1618	* @param pVCpu The cross context virtual CPU structure.
1619	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1620	* out-of-sync. Make sure to update the required fields
1621	* before using them.
1622	*
1623	* @remarks No-long-jump zone!!!
1624	*/
1625	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1626	{
1627	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1628	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1629
1630	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1631	#if HC_ARCH_BITS == 64
1632	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1633	{
1634	/*
1635	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1636	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1637	* we can skip a few MSR writes.
1638	*
1639	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1640	* guest MSR values in the guest-CPU context might be different to what's currently
1641	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1642	* CPU, see @bugref{8728}.
1643	*/
1644	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1645	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1646	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1647	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1648	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1649	{
1650	#ifdef VBOX_STRICT
1651	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1652	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1653	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1654	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1655	#endif
1656	}
1657	else
1658	{
1659	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1660	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1661	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1662	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1663	}
1664	}
1665	#else
1666	RT_NOREF(pMixedCtx);
1667	#endif
1668	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1669	}
1670
1671
1672	/**
1673	* Performs lazy restoration of the set of host MSRs if they were previously
1674	* loaded with guest MSR values.
1675	*
1676	* @param pVCpu The cross context virtual CPU structure.
1677	*
1678	* @remarks No-long-jump zone!!!
1679	* @remarks The guest MSRs should have been saved back into the guest-CPU
1680	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1681	*/
1682	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1683	{
1684	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1685	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1686
1687	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1688	{
1689	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1690	#if HC_ARCH_BITS == 64
1691	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1692	{
1693	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1694	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1695	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1696	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1697	}
1698	#endif
1699	}
1700	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1701	}
1702
1703
1704	/**
1705	* Verifies that our cached values of the VMCS controls are all
1706	* consistent with what's actually present in the VMCS.
1707	*
1708	* @returns VBox status code.
1709	* @param pVCpu The cross context virtual CPU structure.
1710	*/
1711	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1712	{
1713	uint32_t u32Val;
1714	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1715	AssertRCReturn(rc, rc);
1716	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1717	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1718
1719	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1720	AssertRCReturn(rc, rc);
1721	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1722	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1723
1724	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1725	AssertRCReturn(rc, rc);
1726	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1727	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1728
1729	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1730	AssertRCReturn(rc, rc);
1731	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1732	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1733
1734	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1735	{
1736	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1737	AssertRCReturn(rc, rc);
1738	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1739	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1740	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1741	}
1742
1743	return VINF_SUCCESS;
1744	}
1745
1746
1747	#ifdef VBOX_STRICT
1748	/**
1749	* Verifies that our cached host EFER value has not changed
1750	* since we cached it.
1751	*
1752	* @param pVCpu The cross context virtual CPU structure.
1753	*/
1754	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1755	{
1756	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1757
1758	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1759	{
1760	uint64_t u64Val;
1761	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1762	AssertRC(rc);
1763
1764	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1765	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1766	}
1767	}
1768
1769
1770	/**
1771	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1772	* VMCS are correct.
1773	*
1774	* @param pVCpu The cross context virtual CPU structure.
1775	*/
1776	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1777	{
1778	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1779
1780	/* Verify MSR counts in the VMCS are what we think it should be. */
1781	uint32_t cMsrs;
1782	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1783	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1784
1785	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1786	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1787
1788	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1789	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1790
1791	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1792	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1793	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1794	{
1795	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1796	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1797	pGuestMsr->u32Msr, cMsrs));
1798
1799	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1800	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1801	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1802
1803	/* Verify that the permissions are as expected in the MSR bitmap. */
1804	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1805	{
1806	VMXMSREXITREAD enmRead;
1807	VMXMSREXITWRITE enmWrite;
1808	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1809	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1810	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1811	{
1812	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1813	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1814	}
1815	else
1816	{
1817	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1818	pGuestMsr->u32Msr, cMsrs));
1819	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1820	pGuestMsr->u32Msr, cMsrs));
1821	}
1822	}
1823	}
1824	}
1825	#endif /* VBOX_STRICT */
1826
1827
1828	/**
1829	* Flushes the TLB using EPT.
1830	*
1831	* @returns VBox status code.
1832	* @param pVCpu The cross context virtual CPU structure of the calling
1833	* EMT. Can be NULL depending on @a enmFlush.
1834	* @param enmFlush Type of flush.
1835	*
1836	* @remarks Caller is responsible for making sure this function is called only
1837	* when NestedPaging is supported and providing @a enmFlush that is
1838	* supported by the CPU.
1839	* @remarks Can be called with interrupts disabled.
1840	*/
1841	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1842	{
1843	uint64_t au64Descriptor[2];
1844	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1845	au64Descriptor[0] = 0;
1846	else
1847	{
1848	Assert(pVCpu);
1849	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1850	}
1851	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1852
1853	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1854	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1855	rc));
1856	if ( RT_SUCCESS(rc)
1857	&& pVCpu)
1858	{
1859	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1860	}
1861	}
1862
1863
1864	/**
1865	* Flushes the TLB using VPID.
1866	*
1867	* @returns VBox status code.
1868	* @param pVM The cross context VM structure.
1869	* @param pVCpu The cross context virtual CPU structure of the calling
1870	* EMT. Can be NULL depending on @a enmFlush.
1871	* @param enmFlush Type of flush.
1872	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1873	* on @a enmFlush).
1874	*
1875	* @remarks Can be called with interrupts disabled.
1876	*/
1877	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1878	{
1879	NOREF(pVM);
1880	AssertPtr(pVM);
1881	Assert(pVM->hm.s.vmx.fVpid);
1882
1883	uint64_t au64Descriptor[2];
1884	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1885	{
1886	au64Descriptor[0] = 0;
1887	au64Descriptor[1] = 0;
1888	}
1889	else
1890	{
1891	AssertPtr(pVCpu);
1892	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1893	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1894	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1895	au64Descriptor[1] = GCPtr;
1896	}
1897
1898	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1899	AssertMsg(rc == VINF_SUCCESS,
1900	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1901	if ( RT_SUCCESS(rc)
1902	&& pVCpu)
1903	{
1904	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1905	}
1906	}
1907
1908
1909	/**
1910	* Invalidates a guest page by guest virtual address. Only relevant for
1911	* EPT/VPID, otherwise there is nothing really to invalidate.
1912	*
1913	* @returns VBox status code.
1914	* @param pVM The cross context VM structure.
1915	* @param pVCpu The cross context virtual CPU structure.
1916	* @param GCVirt Guest virtual address of the page to invalidate.
1917	*/
1918	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1919	{
1920	AssertPtr(pVM);
1921	AssertPtr(pVCpu);
1922	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1923
1924	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1925	if (!fFlushPending)
1926	{
1927	/*
1928	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1929	* See @bugref{6043} and @bugref{6177}.
1930	*
1931	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1932	* function maybe called in a loop with individual addresses.
1933	*/
1934	if (pVM->hm.s.vmx.fVpid)
1935	{
1936	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1937
1938	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1939	/*
1940	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1941	* where executing INVVPID outside 64-bit mode does not flush translations of
1942	* 64-bit linear addresses, see @bugref{6208#c72}.
1943	*/
1944	if (RT_HI_U32(GCVirt))
1945	fVpidFlush = false;
1946	#endif
1947
1948	if (fVpidFlush)
1949	{
1950	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1951	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1952	}
1953	else
1954	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1955	}
1956	else if (pVM->hm.s.fNestedPaging)
1957	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1958	}
1959
1960	return VINF_SUCCESS;
1961	}
1962
1963
1964	/**
1965	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1966	* case where neither EPT nor VPID is supported by the CPU.
1967	*
1968	* @param pVM The cross context VM structure.
1969	* @param pVCpu The cross context virtual CPU structure.
1970	* @param pCpu Pointer to the global HM struct.
1971	*
1972	* @remarks Called with interrupts disabled.
1973	*/
1974	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1975	{
1976	AssertPtr(pVCpu);
1977	AssertPtr(pCpu);
1978	NOREF(pVM);
1979
1980	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1981
1982	Assert(pCpu->idCpu != NIL_RTCPUID);
1983	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1984	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1985	pVCpu->hm.s.fForceTLBFlush = false;
1986	return;
1987	}
1988
1989
1990	/**
1991	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1992	*
1993	* @param pVM The cross context VM structure.
1994	* @param pVCpu The cross context virtual CPU structure.
1995	* @param pCpu Pointer to the global HM CPU struct.
1996	* @remarks All references to "ASID" in this function pertains to "VPID" in
1997	* Intel's nomenclature. The reason is, to avoid confusion in compare
1998	* statements since the host-CPU copies are named "ASID".
1999	*
2000	* @remarks Called with interrupts disabled.
2001	*/
2002	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2003	{
2004	#ifdef VBOX_WITH_STATISTICS
2005	bool fTlbFlushed = false;
2006	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2007	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2008	if (!fTlbFlushed) \
2009	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2010	} while (0)
2011	#else
2012	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2013	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2014	#endif
2015
2016	AssertPtr(pVM);
2017	AssertPtr(pCpu);
2018	AssertPtr(pVCpu);
2019	Assert(pCpu->idCpu != NIL_RTCPUID);
2020
2021	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2022	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2023	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2024
2025	/*
2026	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2027	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2028	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2029	*/
2030	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2031	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2032	{
2033	++pCpu->uCurrentAsid;
2034	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2035	{
2036	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2037	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2038	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2039	}
2040
2041	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2042	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2043	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2044
2045	/*
2046	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2047	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2048	*/
2049	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2050	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2051	HMVMX_SET_TAGGED_TLB_FLUSHED();
2052	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2053	}
2054
2055	/* Check for explicit TLB flushes. */
2056	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2057	{
2058	/*
2059	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2060	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2061	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2062	* but not guest-physical mappings.
2063	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2064	*/
2065	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2066	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2067	HMVMX_SET_TAGGED_TLB_FLUSHED();
2068	}
2069
2070	pVCpu->hm.s.fForceTLBFlush = false;
2071	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2072
2073	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2074	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2075	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2076	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2077	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2078	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2079	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2080	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2081	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2082
2083	/* Update VMCS with the VPID. */
2084	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2085	AssertRC(rc);
2086
2087	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2088	}
2089
2090
2091	/**
2092	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2093	*
2094	* @returns VBox status code.
2095	* @param pVM The cross context VM structure.
2096	* @param pVCpu The cross context virtual CPU structure.
2097	* @param pCpu Pointer to the global HM CPU struct.
2098	*
2099	* @remarks Called with interrupts disabled.
2100	*/
2101	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2102	{
2103	AssertPtr(pVM);
2104	AssertPtr(pVCpu);
2105	AssertPtr(pCpu);
2106	Assert(pCpu->idCpu != NIL_RTCPUID);
2107	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2108	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2109
2110	/*
2111	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2112	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2113	*/
2114	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2115	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2116	{
2117	pVCpu->hm.s.fForceTLBFlush = true;
2118	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2119	}
2120
2121	/* Check for explicit TLB flushes. */
2122	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2123	{
2124	pVCpu->hm.s.fForceTLBFlush = true;
2125	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2126	}
2127
2128	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2129	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2130
2131	if (pVCpu->hm.s.fForceTLBFlush)
2132	{
2133	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2134	pVCpu->hm.s.fForceTLBFlush = false;
2135	}
2136	}
2137
2138
2139	/**
2140	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2141	*
2142	* @returns VBox status code.
2143	* @param pVM The cross context VM structure.
2144	* @param pVCpu The cross context virtual CPU structure.
2145	* @param pCpu Pointer to the global HM CPU struct.
2146	*
2147	* @remarks Called with interrupts disabled.
2148	*/
2149	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2150	{
2151	AssertPtr(pVM);
2152	AssertPtr(pVCpu);
2153	AssertPtr(pCpu);
2154	Assert(pCpu->idCpu != NIL_RTCPUID);
2155	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2156	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2157
2158	/*
2159	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2160	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2161	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2162	*/
2163	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2164	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2165	{
2166	pVCpu->hm.s.fForceTLBFlush = true;
2167	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2168	}
2169
2170	/* Check for explicit TLB flushes. */
2171	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2172	{
2173	/*
2174	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2175	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2176	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2177	*/
2178	pVCpu->hm.s.fForceTLBFlush = true;
2179	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2180	}
2181
2182	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2183	if (pVCpu->hm.s.fForceTLBFlush)
2184	{
2185	++pCpu->uCurrentAsid;
2186	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2187	{
2188	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2189	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2190	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2191	}
2192
2193	pVCpu->hm.s.fForceTLBFlush = false;
2194	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2195	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2196	if (pCpu->fFlushAsidBeforeUse)
2197	{
2198	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2199	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2200	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2201	{
2202	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2203	pCpu->fFlushAsidBeforeUse = false;
2204	}
2205	else
2206	{
2207	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2208	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2209	}
2210	}
2211	}
2212
2213	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2214	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2215	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2216	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2217	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2218	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2219	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2220
2221	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2222	AssertRC(rc);
2223	}
2224
2225
2226	/**
2227	* Flushes the guest TLB entry based on CPU capabilities.
2228	*
2229	* @param pVCpu The cross context virtual CPU structure.
2230	* @param pCpu Pointer to the global HM CPU struct.
2231	*/
2232	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2233	{
2234	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2235	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2236	#endif
2237	PVM pVM = pVCpu->CTX_SUFF(pVM);
2238	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2239	{
2240	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2241	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2242	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2243	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2244	default:
2245	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2246	break;
2247	}
2248
2249	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2250	}
2251
2252
2253	/**
2254	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2255	* TLB entries from the host TLB before VM-entry.
2256	*
2257	* @returns VBox status code.
2258	* @param pVM The cross context VM structure.
2259	*/
2260	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2261	{
2262	/*
2263	* Determine optimal flush type for Nested Paging.
2264	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2265	* guest execution (see hmR3InitFinalizeR0()).
2266	*/
2267	if (pVM->hm.s.fNestedPaging)
2268	{
2269	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2270	{
2271	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2272	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2273	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2274	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2275	else
2276	{
2277	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2278	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2279	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2280	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2281	}
2282
2283	/* Make sure the write-back cacheable memory type for EPT is supported. */
2284	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2285	{
2286	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2287	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2288	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2289	}
2290
2291	/* EPT requires a page-walk length of 4. */
2292	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2293	{
2294	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2295	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2296	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2297	}
2298	}
2299	else
2300	{
2301	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2302	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2303	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2304	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2305	}
2306	}
2307
2308	/*
2309	* Determine optimal flush type for VPID.
2310	*/
2311	if (pVM->hm.s.vmx.fVpid)
2312	{
2313	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2314	{
2315	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2316	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2317	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2318	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2319	else
2320	{
2321	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2322	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2323	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2324	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2325	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2326	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2327	pVM->hm.s.vmx.fVpid = false;
2328	}
2329	}
2330	else
2331	{
2332	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2333	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2334	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2335	pVM->hm.s.vmx.fVpid = false;
2336	}
2337	}
2338
2339	/*
2340	* Setup the handler for flushing tagged-TLBs.
2341	*/
2342	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2343	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2344	else if (pVM->hm.s.fNestedPaging)
2345	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2346	else if (pVM->hm.s.vmx.fVpid)
2347	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2348	else
2349	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2350	return VINF_SUCCESS;
2351	}
2352
2353
2354	/**
2355	* Sets up pin-based VM-execution controls in the VMCS.
2356	*
2357	* @returns VBox status code.
2358	* @param pVM The cross context VM structure.
2359	* @param pVCpu The cross context virtual CPU structure.
2360	*/
2361	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2362	{
2363	AssertPtr(pVM);
2364	AssertPtr(pVCpu);
2365
2366	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2367	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2368
2369	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2370	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2371
2372	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2373	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2374
2375	/* Enable the VMX preemption timer. */
2376	if (pVM->hm.s.vmx.fUsePreemptTimer)
2377	{
2378	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2379	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2380	}
2381
2382	#if 0
2383	/* Enable posted-interrupt processing. */
2384	if (pVM->hm.s.fPostedIntrs)
2385	{
2386	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2387	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2388	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2389	}
2390	#endif
2391
2392	if ((val & zap) != val)
2393	{
2394	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2395	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2396	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2397	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2398	}
2399
2400	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2401	AssertRCReturn(rc, rc);
2402
2403	pVCpu->hm.s.vmx.u32PinCtls = val;
2404	return rc;
2405	}
2406
2407
2408	/**
2409	* Sets up processor-based VM-execution controls in the VMCS.
2410	*
2411	* @returns VBox status code.
2412	* @param pVM The cross context VM structure.
2413	* @param pVCpu The cross context virtual CPU structure.
2414	*/
2415	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2416	{
2417	AssertPtr(pVM);
2418	AssertPtr(pVCpu);
2419
2420	int rc = VERR_INTERNAL_ERROR_5;
2421	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2422	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2423
2424	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2425	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2426	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2427	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2428	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2429	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2430	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2431
2432	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2433	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2434	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2435	{
2436	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2437	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2438	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2439	}
2440
2441	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2442	if (!pVM->hm.s.fNestedPaging)
2443	{
2444	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2445	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2446	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2447	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2448	}
2449
2450	/* Use TPR shadowing if supported by the CPU. */
2451	if ( PDMHasApic(pVM)
2452	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2453	{
2454	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2455	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2456	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2457	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2458	AssertRCReturn(rc, rc);
2459
2460	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2461	/* CR8 writes cause a VM-exit based on TPR threshold. */
2462	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2463	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2464	}
2465	else
2466	{
2467	/*
2468	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2469	* Set this control only for 64-bit guests.
2470	*/
2471	if (pVM->hm.s.fAllow64BitGuests)
2472	{
2473	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2474	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2475	}
2476	}
2477
2478	/* Use MSR-bitmaps if supported by the CPU. */
2479	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2480	{
2481	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2482
2483	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2484	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2485	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2486	AssertRCReturn(rc, rc);
2487
2488	/*
2489	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2490	* automatically using dedicated fields in the VMCS.
2491	*/
2492	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2493	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2494	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2495	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2496	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2497
2498	#if HC_ARCH_BITS == 64
2499	/*
2500	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2501	*/
2502	if (pVM->hm.s.fAllow64BitGuests)
2503	{
2504	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2508	}
2509	#endif
2510	/*
2511	* The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2512	* access (writes need to be executed without exiting, reds will #GP-fault anyway).
2513	*/
2514	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2515	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516
2517	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2518	}
2519
2520	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2521	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2522	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2523
2524	if ((val & zap) != val)
2525	{
2526	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2527	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2528	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2529	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2530	}
2531
2532	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2533	AssertRCReturn(rc, rc);
2534
2535	pVCpu->hm.s.vmx.u32ProcCtls = val;
2536
2537	/*
2538	* Secondary processor-based VM-execution controls.
2539	*/
2540	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2541	{
2542	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2543	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2544
2545	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2546	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2547
2548	if (pVM->hm.s.fNestedPaging)
2549	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2550
2551	/*
2552	* Enable the INVPCID instruction if supported by the hardware and we expose
2553	* it to the guest. Without this, guest executing INVPCID would cause a #UD.
2554	*/
2555	if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2556	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2557	{
2558	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2559	}
2560
2561	if (pVM->hm.s.vmx.fVpid)
2562	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2563
2564	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2565	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2566
2567	#if 0
2568	if (pVM->hm.s.fVirtApicRegs)
2569	{
2570	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2571	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2572
2573	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2574	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2575	}
2576	#endif
2577
2578	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2579	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2580	* done dynamically. */
2581	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2582	{
2583	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2584	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2585	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2586	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2587	AssertRCReturn(rc, rc);
2588	}
2589
2590	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2591	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2592
2593	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2594	&& pVM->hm.s.vmx.cPleGapTicks
2595	&& pVM->hm.s.vmx.cPleWindowTicks)
2596	{
2597	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2598
2599	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2600	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2601	AssertRCReturn(rc, rc);
2602	}
2603
2604	if ((val & zap) != val)
2605	{
2606	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2607	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2608	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2609	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2610	}
2611
2612	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2613	AssertRCReturn(rc, rc);
2614
2615	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2616	}
2617	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2618	{
2619	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2620	"available\n"));
2621	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2622	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2623	}
2624
2625	return VINF_SUCCESS;
2626	}
2627
2628
2629	/**
2630	* Sets up miscellaneous (everything other than Pin & Processor-based
2631	* VM-execution) control fields in the VMCS.
2632	*
2633	* @returns VBox status code.
2634	* @param pVM The cross context VM structure.
2635	* @param pVCpu The cross context virtual CPU structure.
2636	*/
2637	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2638	{
2639	NOREF(pVM);
2640	AssertPtr(pVM);
2641	AssertPtr(pVCpu);
2642
2643	int rc = VERR_GENERAL_FAILURE;
2644
2645	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2646	#if 0
2647	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2648	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2649	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2650
2651	/*
2652	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2653	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2654	* We thus use the exception bitmap to control it rather than use both.
2655	*/
2656	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2657	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2658
2659	/** @todo Explore possibility of using IO-bitmaps. */
2660	/* All IO & IOIO instructions cause VM-exits. */
2661	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2662	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2663
2664	/* Initialize the MSR-bitmap area. */
2665	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2666	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2667	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2668	AssertRCReturn(rc, rc);
2669	#endif
2670
2671	/* Setup MSR auto-load/store area. */
2672	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2673	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2674	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2675	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2676	AssertRCReturn(rc, rc);
2677
2678	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2679	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2680	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2681	AssertRCReturn(rc, rc);
2682
2683	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2684	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2685	AssertRCReturn(rc, rc);
2686
2687	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2688	#if 0
2689	/* Setup debug controls */
2690	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2691	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2692	AssertRCReturn(rc, rc);
2693	#endif
2694
2695	return rc;
2696	}
2697
2698
2699	/**
2700	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2701	*
2702	* We shall setup those exception intercepts that don't change during the
2703	* lifetime of the VM here. The rest are done dynamically while loading the
2704	* guest state.
2705	*
2706	* @returns VBox status code.
2707	* @param pVM The cross context VM structure.
2708	* @param pVCpu The cross context virtual CPU structure.
2709	*/
2710	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2711	{
2712	AssertPtr(pVM);
2713	AssertPtr(pVCpu);
2714
2715	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2716
2717	uint32_t u32XcptBitmap = 0;
2718
2719	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2720	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2721
2722	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2723	and writes, and because recursive #DBs can cause the CPU hang, we must always
2724	intercept #DB. */
2725	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2726
2727	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2728	if (!pVM->hm.s.fNestedPaging)
2729	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2730
2731	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2732	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2733	AssertRCReturn(rc, rc);
2734	return rc;
2735	}
2736
2737
2738	/**
2739	* Sets up the initial guest-state mask. The guest-state mask is consulted
2740	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2741	* for the nested virtualization case (as it would cause a VM-exit).
2742	*
2743	* @param pVCpu The cross context virtual CPU structure.
2744	*/
2745	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2746	{
2747	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2748	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2749	return VINF_SUCCESS;
2750	}
2751
2752
2753	/**
2754	* Does per-VM VT-x initialization.
2755	*
2756	* @returns VBox status code.
2757	* @param pVM The cross context VM structure.
2758	*/
2759	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2760	{
2761	LogFlowFunc(("pVM=%p\n", pVM));
2762
2763	int rc = hmR0VmxStructsAlloc(pVM);
2764	if (RT_FAILURE(rc))
2765	{
2766	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2767	return rc;
2768	}
2769
2770	return VINF_SUCCESS;
2771	}
2772
2773
2774	/**
2775	* Does per-VM VT-x termination.
2776	*
2777	* @returns VBox status code.
2778	* @param pVM The cross context VM structure.
2779	*/
2780	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2781	{
2782	LogFlowFunc(("pVM=%p\n", pVM));
2783
2784	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2785	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2786	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2787	#endif
2788	hmR0VmxStructsFree(pVM);
2789	return VINF_SUCCESS;
2790	}
2791
2792
2793	/**
2794	* Sets up the VM for execution under VT-x.
2795	* This function is only called once per-VM during initialization.
2796	*
2797	* @returns VBox status code.
2798	* @param pVM The cross context VM structure.
2799	*/
2800	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2801	{
2802	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2803	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2804
2805	LogFlowFunc(("pVM=%p\n", pVM));
2806
2807	/*
2808	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2809	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2810	*/
2811	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2812	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2813	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2814	{
2815	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2816	return VERR_INTERNAL_ERROR;
2817	}
2818
2819	/* Initialize these always, see hmR3InitFinalizeR0().*/
2820	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2821	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2822
2823	/* Setup the tagged-TLB flush handlers. */
2824	int rc = hmR0VmxSetupTaggedTlb(pVM);
2825	if (RT_FAILURE(rc))
2826	{
2827	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2828	return rc;
2829	}
2830
2831	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2832	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2833	#if HC_ARCH_BITS == 64
2834	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2835	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2836	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2837	{
2838	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2839	}
2840	#endif
2841
2842	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2843	RTCCUINTREG uHostCR4 = ASMGetCR4();
2844	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2845	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2846
2847	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2848	{
2849	PVMCPU pVCpu = &pVM->aCpus[i];
2850	AssertPtr(pVCpu);
2851	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2852
2853	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2854	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2855
2856	/* Set revision dword at the beginning of the VMCS structure. */
2857	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2858
2859	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2860	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2861	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2862	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2863
2864	/* Load this VMCS as the current VMCS. */
2865	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2866	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2867	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2868
2869	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2870	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2871	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2872
2873	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2874	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2875	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2876
2877	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2878	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2879	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2880
2881	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2882	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2883	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2884
2885	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2886	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2887	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2888
2889	#if HC_ARCH_BITS == 32
2890	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2891	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2892	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2893	#endif
2894
2895	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2896	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2897	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2898	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2899
2900	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2901
2902	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2903	}
2904
2905	return VINF_SUCCESS;
2906	}
2907
2908
2909	/**
2910	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2911	* the VMCS.
2912	*
2913	* @returns VBox status code.
2914	* @param pVM The cross context VM structure.
2915	* @param pVCpu The cross context virtual CPU structure.
2916	*/
2917	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2918	{
2919	NOREF(pVM); NOREF(pVCpu);
2920
2921	RTCCUINTREG uReg = ASMGetCR0();
2922	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2923	AssertRCReturn(rc, rc);
2924
2925	uReg = ASMGetCR3();
2926	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2927	AssertRCReturn(rc, rc);
2928
2929	uReg = ASMGetCR4();
2930	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2931	AssertRCReturn(rc, rc);
2932	return rc;
2933	}
2934
2935
2936	#if HC_ARCH_BITS == 64
2937	/**
2938	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2939	* requirements. See hmR0VmxSaveHostSegmentRegs().
2940	*/
2941	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2942	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2943	{ \
2944	bool fValidSelector = true; \
2945	if ((selValue) & X86_SEL_LDT) \
2946	{ \
2947	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2948	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2949	} \
2950	if (fValidSelector) \
2951	{ \
2952	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2953	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2954	} \
2955	(selValue) = 0; \
2956	}
2957	#endif
2958
2959
2960	/**
2961	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2962	* the host-state area in the VMCS.
2963	*
2964	* @returns VBox status code.
2965	* @param pVM The cross context VM structure.
2966	* @param pVCpu The cross context virtual CPU structure.
2967	*/
2968	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2969	{
2970	int rc = VERR_INTERNAL_ERROR_5;
2971
2972	#if HC_ARCH_BITS == 64
2973	/*
2974	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2975	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2976	*
2977	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2978	* Was observed booting Solaris10u10 32-bit guest.
2979	*/
2980	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2981	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2982	{
2983	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2984	pVCpu->idCpu));
2985	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2986	}
2987	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2988	#else
2989	RT_NOREF(pVCpu);
2990	#endif
2991
2992	/*
2993	* Host DS, ES, FS and GS segment registers.
2994	*/
2995	#if HC_ARCH_BITS == 64
2996	RTSEL uSelDS = ASMGetDS();
2997	RTSEL uSelES = ASMGetES();
2998	RTSEL uSelFS = ASMGetFS();
2999	RTSEL uSelGS = ASMGetGS();
3000	#else
3001	RTSEL uSelDS = 0;
3002	RTSEL uSelES = 0;
3003	RTSEL uSelFS = 0;
3004	RTSEL uSelGS = 0;
3005	#endif
3006
3007	/*
3008	* Host CS and SS segment registers.
3009	*/
3010	RTSEL uSelCS = ASMGetCS();
3011	RTSEL uSelSS = ASMGetSS();
3012
3013	/*
3014	* Host TR segment register.
3015	*/
3016	RTSEL uSelTR = ASMGetTR();
3017
3018	#if HC_ARCH_BITS == 64
3019	/*
3020	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3021	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3022	*/
3023	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3024	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3025	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3026	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3027	# undef VMXLOCAL_ADJUST_HOST_SEG
3028	#endif
3029
3030	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3031	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3032	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3033	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3034	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3035	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3036	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3037	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3038	Assert(uSelCS);
3039	Assert(uSelTR);
3040
3041	/* Assertion is right but we would not have updated u32ExitCtls yet. */
3042	#if 0
3043	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3044	Assert(uSelSS != 0);
3045	#endif
3046
3047	/* Write these host selector fields into the host-state area in the VMCS. */
3048	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3049	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3050	#if HC_ARCH_BITS == 64
3051	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3052	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3053	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3054	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3055	#else
3056	NOREF(uSelDS);
3057	NOREF(uSelES);
3058	NOREF(uSelFS);
3059	NOREF(uSelGS);
3060	#endif
3061	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3062	AssertRCReturn(rc, rc);
3063
3064	/*
3065	* Host GDTR and IDTR.
3066	*/
3067	RTGDTR Gdtr;
3068	RTIDTR Idtr;
3069	RT_ZERO(Gdtr);
3070	RT_ZERO(Idtr);
3071	ASMGetGDTR(&Gdtr);
3072	ASMGetIDTR(&Idtr);
3073	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3074	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3075	AssertRCReturn(rc, rc);
3076
3077	#if HC_ARCH_BITS == 64
3078	/*
3079	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3080	* maximum limit (0xffff) on every VM-exit.
3081	*/
3082	if (Gdtr.cbGdt != 0xffff)
3083	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3084
3085	/*
3086	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3087	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3088	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3089	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3090	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3091	* hosts where we are pretty sure it won't cause trouble.
3092	*/
3093	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3094	if (Idtr.cbIdt < 0x0fff)
3095	# else
3096	if (Idtr.cbIdt != 0xffff)
3097	# endif
3098	{
3099	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3100	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3101	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3102	}
3103	#endif
3104
3105	/*
3106	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3107	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3108	*/
3109	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3110	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3111	VERR_VMX_INVALID_HOST_STATE);
3112
3113	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3114	#if HC_ARCH_BITS == 64
3115	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3116
3117	/*
3118	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3119	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3120	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3121	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3122	*
3123	* [1] See Intel spec. 3.5 "System Descriptor Types".
3124	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3125	*/
3126	Assert(pDesc->System.u4Type == 11);
3127	if ( pDesc->System.u16LimitLow != 0x67
3128	\|\| pDesc->System.u4LimitHigh)
3129	{
3130	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3131	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3132	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3133	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3134	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3135	}
3136
3137	/*
3138	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3139	*/
3140	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3141	{
3142	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3143	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3144	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3145	{
3146	/* The GDT is read-only but the writable GDT is available. */
3147	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3148	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3149	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3150	AssertRCReturn(rc, rc);
3151	}
3152	}
3153	#else
3154	NOREF(pVM);
3155	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3156	#endif
3157	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3158	AssertRCReturn(rc, rc);
3159
3160	/*
3161	* Host FS base and GS base.
3162	*/
3163	#if HC_ARCH_BITS == 64
3164	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3165	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3166	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3167	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3168	AssertRCReturn(rc, rc);
3169
3170	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3171	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3172	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3173	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3174	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3175	#endif
3176	return rc;
3177	}
3178
3179
3180	/**
3181	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3182	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3183	* the host after every successful VM-exit.
3184	*
3185	* @returns VBox status code.
3186	* @param pVM The cross context VM structure.
3187	* @param pVCpu The cross context virtual CPU structure.
3188	*
3189	* @remarks No-long-jump zone!!!
3190	*/
3191	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3192	{
3193	NOREF(pVM);
3194
3195	AssertPtr(pVCpu);
3196	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3197
3198	/*
3199	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3200	* rather than swapping them on every VM-entry.
3201	*/
3202	hmR0VmxLazySaveHostMsrs(pVCpu);
3203
3204	/*
3205	* Host Sysenter MSRs.
3206	*/
3207	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3208	#if HC_ARCH_BITS == 32
3209	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3210	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3211	#else
3212	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3213	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3214	#endif
3215	AssertRCReturn(rc, rc);
3216
3217	/*
3218	* Host EFER MSR.
3219	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3220	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3221	*/
3222	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3223	{
3224	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3225	AssertRCReturn(rc, rc);
3226	}
3227
3228	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3229	* hmR0VmxLoadGuestExitCtls() !! */
3230
3231	return rc;
3232	}
3233
3234
3235	/**
3236	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3237	*
3238	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3239	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3240	* hmR0VMxLoadGuestEntryCtls().
3241	*
3242	* @returns true if we need to load guest EFER, false otherwise.
3243	* @param pVCpu The cross context virtual CPU structure.
3244	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3245	* out-of-sync. Make sure to update the required fields
3246	* before using them.
3247	*
3248	* @remarks Requires EFER, CR4.
3249	* @remarks No-long-jump zone!!!
3250	*/
3251	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3252	{
3253	#ifdef HMVMX_ALWAYS_SWAP_EFER
3254	return true;
3255	#endif
3256
3257	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3258	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3259	if (CPUMIsGuestInLongMode(pVCpu))
3260	return false;
3261	#endif
3262
3263	PVM pVM = pVCpu->CTX_SUFF(pVM);
3264	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3265	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3266
3267	/*
3268	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3269	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3270	*/
3271	if ( CPUMIsGuestInLongMode(pVCpu)
3272	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3273	{
3274	return true;
3275	}
3276
3277	/*
3278	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3279	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3280	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3281	*/
3282	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3283	&& (pMixedCtx->cr0 & X86_CR0_PG)
3284	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3285	{
3286	/* Assert that host is PAE capable. */
3287	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3288	return true;
3289	}
3290
3291	/** @todo Check the latest Intel spec. for any other bits,
3292	* like SMEP/SMAP? */
3293	return false;
3294	}
3295
3296
3297	/**
3298	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3299	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3300	* controls".
3301	*
3302	* @returns VBox status code.
3303	* @param pVCpu The cross context virtual CPU structure.
3304	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3305	* out-of-sync. Make sure to update the required fields
3306	* before using them.
3307	*
3308	* @remarks Requires EFER.
3309	* @remarks No-long-jump zone!!!
3310	*/
3311	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3312	{
3313	int rc = VINF_SUCCESS;
3314	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3315	{
3316	PVM pVM = pVCpu->CTX_SUFF(pVM);
3317	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3318	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3319
3320	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3321	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3322
3323	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3324	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3325	{
3326	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3327	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3328	}
3329	else
3330	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3331
3332	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3333	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3334	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3335	{
3336	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3337	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3338	}
3339
3340	/*
3341	* The following should -not- be set (since we're not in SMM mode):
3342	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3343	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3344	*/
3345
3346	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3347	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3348
3349	if ((val & zap) != val)
3350	{
3351	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3352	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3353	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3355	}
3356
3357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3358	AssertRCReturn(rc, rc);
3359
3360	pVCpu->hm.s.vmx.u32EntryCtls = val;
3361	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3362	}
3363	return rc;
3364	}
3365
3366
3367	/**
3368	* Sets up the VM-exit controls in the VMCS.
3369	*
3370	* @returns VBox status code.
3371	* @param pVCpu The cross context virtual CPU structure.
3372	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3373	* out-of-sync. Make sure to update the required fields
3374	* before using them.
3375	*
3376	* @remarks Requires EFER.
3377	*/
3378	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3379	{
3380	NOREF(pMixedCtx);
3381
3382	int rc = VINF_SUCCESS;
3383	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3384	{
3385	PVM pVM = pVCpu->CTX_SUFF(pVM);
3386	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3387	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3388
3389	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3390	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3391
3392	/*
3393	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3394	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3395	*/
3396	#if HC_ARCH_BITS == 64
3397	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3398	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3399	#else
3400	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3401	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3402	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3403	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3404	{
3405	/* The switcher returns to long mode, EFER is managed by the switcher. */
3406	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3407	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3408	}
3409	else
3410	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3411	#endif
3412
3413	/* If the newer VMCS fields for managing EFER exists, use it. */
3414	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3415	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3416	{
3417	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3418	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3419	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3420	}
3421
3422	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3423	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3424
3425	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3426	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3427	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3428
3429	if ( pVM->hm.s.vmx.fUsePreemptTimer
3430	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3431	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3432
3433	if ((val & zap) != val)
3434	{
3435	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3436	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3437	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3438	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3439	}
3440
3441	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3442	AssertRCReturn(rc, rc);
3443
3444	pVCpu->hm.s.vmx.u32ExitCtls = val;
3445	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3446	}
3447	return rc;
3448	}
3449
3450
3451	/**
3452	* Sets the TPR threshold in the VMCS.
3453	*
3454	* @returns VBox status code.
3455	* @param pVCpu The cross context virtual CPU structure.
3456	* @param u32TprThreshold The TPR threshold (task-priority class only).
3457	*/
3458	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3459	{
3460	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3461	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3462	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3463	}
3464
3465
3466	/**
3467	* Loads the guest APIC and related state.
3468	*
3469	* @returns VBox status code.
3470	* @param pVCpu The cross context virtual CPU structure.
3471	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3472	* out-of-sync. Make sure to update the required fields
3473	* before using them.
3474	*
3475	* @remarks No-long-jump zone!!!
3476	*/
3477	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3478	{
3479	NOREF(pMixedCtx);
3480
3481	int rc = VINF_SUCCESS;
3482	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_APIC_STATE))
3483	{
3484	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3485	&& APICIsEnabled(pVCpu))
3486	{
3487	/*
3488	* Setup TPR shadowing.
3489	*/
3490	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3491	{
3492	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3493
3494	bool fPendingIntr = false;
3495	uint8_t u8Tpr = 0;
3496	uint8_t u8PendingIntr = 0;
3497	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3498	AssertRCReturn(rc, rc);
3499
3500	/*
3501	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3502	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3503	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3504	*/
3505	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3506	uint32_t u32TprThreshold = 0;
3507	if (fPendingIntr)
3508	{
3509	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3510	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3511	const uint8_t u8TprPriority = u8Tpr >> 4;
3512	if (u8PendingPriority <= u8TprPriority)
3513	u32TprThreshold = u8PendingPriority;
3514	}
3515
3516	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3517	AssertRCReturn(rc, rc);
3518	}
3519	}
3520	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
3521	}
3522
3523	return rc;
3524	}
3525
3526
3527	/**
3528	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3529	*
3530	* @returns Guest's interruptibility-state.
3531	* @param pVCpu The cross context virtual CPU structure.
3532	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3533	* out-of-sync. Make sure to update the required fields
3534	* before using them.
3535	*
3536	* @remarks No-long-jump zone!!!
3537	*/
3538	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3539	{
3540	/*
3541	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3542	*/
3543	uint32_t uIntrState = 0;
3544	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3545	{
3546	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3547	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3548	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3549	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3550	{
3551	if (pMixedCtx->eflags.Bits.u1IF)
3552	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3553	else
3554	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3555	}
3556	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3557	{
3558	/*
3559	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3560	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3561	*/
3562	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3563	}
3564	}
3565
3566	/*
3567	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3568	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3569	* setting this would block host-NMIs and IRET will not clear the blocking.
3570	*
3571	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3572	*/
3573	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3574	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3575	{
3576	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3577	}
3578
3579	return uIntrState;
3580	}
3581
3582
3583	/**
3584	* Loads the guest's interruptibility-state into the guest-state area in the
3585	* VMCS.
3586	*
3587	* @returns VBox status code.
3588	* @param pVCpu The cross context virtual CPU structure.
3589	* @param uIntrState The interruptibility-state to set.
3590	*/
3591	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3592	{
3593	NOREF(pVCpu);
3594	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3595	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3596	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3597	AssertRC(rc);
3598	return rc;
3599	}
3600
3601
3602	/**
3603	* Loads the exception intercepts required for guest execution in the VMCS.
3604	*
3605	* @returns VBox status code.
3606	* @param pVCpu The cross context virtual CPU structure.
3607	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3608	* out-of-sync. Make sure to update the required fields
3609	* before using them.
3610	*/
3611	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3612	{
3613	NOREF(pMixedCtx);
3614	int rc = VINF_SUCCESS;
3615	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
3616	{
3617	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3618	if (pVCpu->hm.s.fGIMTrapXcptUD)
3619	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3620	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3621	else
3622	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3623	#endif
3624
3625	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3626	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3627
3628	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3629	AssertRCReturn(rc, rc);
3630
3631	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3632	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3633	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3634	}
3635	return rc;
3636	}
3637
3638
3639	/**
3640	* Loads the guest's RIP into the guest-state area in the VMCS.
3641	*
3642	* @returns VBox status code.
3643	* @param pVCpu The cross context virtual CPU structure.
3644	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3645	* out-of-sync. Make sure to update the required fields
3646	* before using them.
3647	*
3648	* @remarks No-long-jump zone!!!
3649	*/
3650	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3651	{
3652	int rc = VINF_SUCCESS;
3653	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3654	{
3655	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3656	AssertRCReturn(rc, rc);
3657
3658	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3659	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3660	HMCPU_CF_VALUE(pVCpu)));
3661
3662	/* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3663	if (HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3664	EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3665	else
3666	EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3667	}
3668	return rc;
3669	}
3670
3671
3672	/**
3673	* Loads the guest's RSP into the guest-state area in the VMCS.
3674	*
3675	* @returns VBox status code.
3676	* @param pVCpu The cross context virtual CPU structure.
3677	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3678	* out-of-sync. Make sure to update the required fields
3679	* before using them.
3680	*
3681	* @remarks No-long-jump zone!!!
3682	*/
3683	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3684	{
3685	int rc = VINF_SUCCESS;
3686	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3687	{
3688	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3689	AssertRCReturn(rc, rc);
3690
3691	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3692	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3693	}
3694	return rc;
3695	}
3696
3697
3698	/**
3699	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3700	*
3701	* @returns VBox status code.
3702	* @param pVCpu The cross context virtual CPU structure.
3703	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3704	* out-of-sync. Make sure to update the required fields
3705	* before using them.
3706	*
3707	* @remarks No-long-jump zone!!!
3708	*/
3709	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3710	{
3711	int rc = VINF_SUCCESS;
3712	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3713	{
3714	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3715	Let us assert it as such and use 32-bit VMWRITE. */
3716	Assert(!(pMixedCtx->rflags.u64 >> 32));
3717	X86EFLAGS Eflags = pMixedCtx->eflags;
3718	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3719	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3720	* These will never be cleared/set, unless some other part of the VMM
3721	* code is buggy - in which case we're better of finding and fixing
3722	* those bugs than hiding them. */
3723	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3724	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3725	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3726	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3727
3728	/*
3729	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3730	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3731	*/
3732	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3733	{
3734	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3735	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3736	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3737	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3738	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3739	}
3740
3741	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3742	AssertRCReturn(rc, rc);
3743
3744	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3745	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3746	}
3747	return rc;
3748	}
3749
3750
3751	/**
3752	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3753	*
3754	* @returns VBox status code.
3755	* @param pVCpu The cross context virtual CPU structure.
3756	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3757	* out-of-sync. Make sure to update the required fields
3758	* before using them.
3759	*
3760	* @remarks No-long-jump zone!!!
3761	*/
3762	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3763	{
3764	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3765	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3766	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3767	AssertRCReturn(rc, rc);
3768	return rc;
3769	}
3770
3771
3772	/**
3773	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3774	* CR0 is partially shared with the host and we have to consider the FPU bits.
3775	*
3776	* @returns VBox status code.
3777	* @param pVCpu The cross context virtual CPU structure.
3778	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3779	* out-of-sync. Make sure to update the required fields
3780	* before using them.
3781	*
3782	* @remarks No-long-jump zone!!!
3783	*/
3784	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3785	{
3786	Assert(CPUMIsGuestFPUStateActive(pVCpu));
3787
3788	/*
3789	* Guest CR0.
3790	* Guest FPU.
3791	*/
3792	int rc = VINF_SUCCESS;
3793	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3794	{
3795	Assert(!(pMixedCtx->cr0 >> 32));
3796	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3797	PVM pVM = pVCpu->CTX_SUFF(pVM);
3798
3799	/* The guest's view (read access) of its CR0 is unblemished. */
3800	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3801	AssertRCReturn(rc, rc);
3802	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3803
3804	/* Setup VT-x's view of the guest CR0. */
3805	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3806	if (pVM->hm.s.fNestedPaging)
3807	{
3808	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3809	{
3810	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3811	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3812	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3813	}
3814	else
3815	{
3816	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3817	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3818	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3819	}
3820
3821	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3822	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3823	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3824
3825	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3826	AssertRCReturn(rc, rc);
3827	}
3828	else
3829	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3830
3831	/*
3832	* Guest FPU bits.
3833	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3834	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3835	*/
3836	u32GuestCR0 \|= X86_CR0_NE;
3837
3838	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3839	bool fInterceptMF = false;
3840	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3841	fInterceptMF = true;
3842
3843	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3844	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3845	{
3846	Assert(PDMVmmDevHeapIsEnabled(pVM));
3847	Assert(pVM->hm.s.vmx.pRealModeTSS);
3848	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3849	}
3850	else
3851	{
3852	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3853	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3854	if (fInterceptMF)
3855	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3856	}
3857	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3858
3859	/* Additional intercepts for debugging, define these yourself explicitly. */
3860	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3861	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3862	\| RT_BIT(X86_XCPT_BP)
3863	\| RT_BIT(X86_XCPT_DE)
3864	\| RT_BIT(X86_XCPT_NM)
3865	\| RT_BIT(X86_XCPT_TS)
3866	\| RT_BIT(X86_XCPT_UD)
3867	\| RT_BIT(X86_XCPT_NP)
3868	\| RT_BIT(X86_XCPT_SS)
3869	\| RT_BIT(X86_XCPT_GP)
3870	\| RT_BIT(X86_XCPT_PF)
3871	\| RT_BIT(X86_XCPT_MF)
3872	;
3873	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3874	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3875	#endif
3876
3877	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3878
3879	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3880	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3881	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3882	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3883	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3884	else
3885	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3886
3887	u32GuestCR0 \|= uSetCR0;
3888	u32GuestCR0 &= uZapCR0;
3889	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3890
3891	/* Write VT-x's view of the guest CR0 into the VMCS. */
3892	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3893	AssertRCReturn(rc, rc);
3894	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3895	uZapCR0));
3896
3897	/*
3898	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3899	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3900	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3901	*/
3902	uint32_t u32CR0Mask = 0;
3903	u32CR0Mask = X86_CR0_PE
3904	\| X86_CR0_NE
3905	\| X86_CR0_WP
3906	\| X86_CR0_PG
3907	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3908	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3909	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3910
3911	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3912	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3913	* and @bugref{6944}. */
3914	#if 0
3915	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3916	u32CR0Mask &= ~X86_CR0_PE;
3917	#endif
3918	if (pVM->hm.s.fNestedPaging)
3919	u32CR0Mask &= ~X86_CR0_WP;
3920
3921	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3922	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3923	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3924	AssertRCReturn(rc, rc);
3925	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3926
3927	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3928	}
3929	return rc;
3930	}
3931
3932
3933	/**
3934	* Loads the guest control registers (CR3, CR4) into the guest-state area
3935	* in the VMCS.
3936	*
3937	* @returns VBox strict status code.
3938	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3939	* without unrestricted guest access and the VMMDev is not presently
3940	* mapped (e.g. EFI32).
3941	*
3942	* @param pVCpu The cross context virtual CPU structure.
3943	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3944	* out-of-sync. Make sure to update the required fields
3945	* before using them.
3946	*
3947	* @remarks No-long-jump zone!!!
3948	*/
3949	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3950	{
3951	int rc = VINF_SUCCESS;
3952	PVM pVM = pVCpu->CTX_SUFF(pVM);
3953
3954	/*
3955	* Guest CR2.
3956	* It's always loaded in the assembler code. Nothing to do here.
3957	*/
3958
3959	/*
3960	* Guest CR3.
3961	*/
3962	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3963	{
3964	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3965	if (pVM->hm.s.fNestedPaging)
3966	{
3967	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3968
3969	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3970	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3971	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3972	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3973
3974	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3975	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3976	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3977
3978	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3979	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3980	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3981	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3982	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3983	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3984	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3985
3986	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3987	AssertRCReturn(rc, rc);
3988	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3989
3990	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3991	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3992	{
3993	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3994	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3995	{
3996	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3997	AssertRCReturn(rc, rc);
3998	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3999	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4000	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4001	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4002	AssertRCReturn(rc, rc);
4003	}
4004
4005	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4006	have Unrestricted Execution to handle the guest when it's not using paging. */
4007	GCPhysGuestCR3 = pMixedCtx->cr3;
4008	}
4009	else
4010	{
4011	/*
4012	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4013	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4014	* EPT takes care of translating it to host-physical addresses.
4015	*/
4016	RTGCPHYS GCPhys;
4017	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4018
4019	/* We obtain it here every time as the guest could have relocated this PCI region. */
4020	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4021	if (RT_SUCCESS(rc))
4022	{ /* likely */ }
4023	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4024	{
4025	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4026	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4027	}
4028	else
4029	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4030
4031	GCPhysGuestCR3 = GCPhys;
4032	}
4033
4034	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4035	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4036	}
4037	else
4038	{
4039	/* Non-nested paging case, just use the hypervisor's CR3. */
4040	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4041
4042	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4043	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4044	}
4045	AssertRCReturn(rc, rc);
4046
4047	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4048	}
4049
4050	/*
4051	* Guest CR4.
4052	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4053	*/
4054	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4055	{
4056	Assert(!(pMixedCtx->cr4 >> 32));
4057	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4058
4059	/* The guest's view of its CR4 is unblemished. */
4060	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4061	AssertRCReturn(rc, rc);
4062	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4063
4064	/* Setup VT-x's view of the guest CR4. */
4065	/*
4066	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4067	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4068	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4069	*/
4070	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4071	{
4072	Assert(pVM->hm.s.vmx.pRealModeTSS);
4073	Assert(PDMVmmDevHeapIsEnabled(pVM));
4074	u32GuestCR4 &= ~X86_CR4_VME;
4075	}
4076
4077	if (pVM->hm.s.fNestedPaging)
4078	{
4079	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4080	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4081	{
4082	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4083	u32GuestCR4 \|= X86_CR4_PSE;
4084	/* Our identity mapping is a 32-bit page directory. */
4085	u32GuestCR4 &= ~X86_CR4_PAE;
4086	}
4087	/* else use guest CR4.*/
4088	}
4089	else
4090	{
4091	/*
4092	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4093	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4094	*/
4095	switch (pVCpu->hm.s.enmShadowMode)
4096	{
4097	case PGMMODE_REAL: /* Real-mode. */
4098	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4099	case PGMMODE_32_BIT: /* 32-bit paging. */
4100	{
4101	u32GuestCR4 &= ~X86_CR4_PAE;
4102	break;
4103	}
4104
4105	case PGMMODE_PAE: /* PAE paging. */
4106	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4107	{
4108	u32GuestCR4 \|= X86_CR4_PAE;
4109	break;
4110	}
4111
4112	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4113	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4114	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4115	break;
4116	#endif
4117	default:
4118	AssertFailed();
4119	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4120	}
4121	}
4122
4123	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4124	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4125	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4126	u32GuestCR4 \|= uSetCR4;
4127	u32GuestCR4 &= uZapCR4;
4128
4129	/* Write VT-x's view of the guest CR4 into the VMCS. */
4130	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4131	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4132	AssertRCReturn(rc, rc);
4133
4134	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4135	uint32_t u32CR4Mask = X86_CR4_VME
4136	\| X86_CR4_PAE
4137	\| X86_CR4_PGE
4138	\| X86_CR4_PSE
4139	\| X86_CR4_VMXE;
4140	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4141	u32CR4Mask \|= X86_CR4_OSXSAVE;
4142	if (pVM->cpum.ro.GuestFeatures.fPcid)
4143	u32CR4Mask \|= X86_CR4_PCIDE;
4144	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4145	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4146	AssertRCReturn(rc, rc);
4147
4148	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4149	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4150
4151	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4152	}
4153	return rc;
4154	}
4155
4156
4157	/**
4158	* Loads the guest debug registers into the guest-state area in the VMCS.
4159	*
4160	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4161	*
4162	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4163	*
4164	* @returns VBox status code.
4165	* @param pVCpu The cross context virtual CPU structure.
4166	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4167	* out-of-sync. Make sure to update the required fields
4168	* before using them.
4169	*
4170	* @remarks No-long-jump zone!!!
4171	*/
4172	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4173	{
4174	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4175	return VINF_SUCCESS;
4176
4177	#ifdef VBOX_STRICT
4178	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4179	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4180	{
4181	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4182	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4183	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4184	}
4185	#endif
4186
4187	int rc;
4188	PVM pVM = pVCpu->CTX_SUFF(pVM);
4189	bool fSteppingDB = false;
4190	bool fInterceptMovDRx = false;
4191	if (pVCpu->hm.s.fSingleInstruction)
4192	{
4193	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4194	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4195	{
4196	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4197	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4198	AssertRCReturn(rc, rc);
4199	Assert(fSteppingDB == false);
4200	}
4201	else
4202	{
4203	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4204	pVCpu->hm.s.fClearTrapFlag = true;
4205	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4206	fSteppingDB = true;
4207	}
4208	}
4209
4210	if ( fSteppingDB
4211	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4212	{
4213	/*
4214	* Use the combined guest and host DRx values found in the hypervisor
4215	* register set because the debugger has breakpoints active or someone
4216	* is single stepping on the host side without a monitor trap flag.
4217	*
4218	* Note! DBGF expects a clean DR6 state before executing guest code.
4219	*/
4220	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4221	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4222	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4223	{
4224	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4225	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4226	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4227	}
4228	else
4229	#endif
4230	if (!CPUMIsHyperDebugStateActive(pVCpu))
4231	{
4232	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4233	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4234	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4235	}
4236
4237	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4238	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4239	AssertRCReturn(rc, rc);
4240
4241	pVCpu->hm.s.fUsingHyperDR7 = true;
4242	fInterceptMovDRx = true;
4243	}
4244	else
4245	{
4246	/*
4247	* If the guest has enabled debug registers, we need to load them prior to
4248	* executing guest code so they'll trigger at the right time.
4249	*/
4250	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4251	{
4252	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4253	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4254	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4255	{
4256	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4257	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4258	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4259	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4260	}
4261	else
4262	#endif
4263	if (!CPUMIsGuestDebugStateActive(pVCpu))
4264	{
4265	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4266	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4267	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4268	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4269	}
4270	Assert(!fInterceptMovDRx);
4271	}
4272	/*
4273	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4274	* must intercept #DB in order to maintain a correct DR6 guest value, and
4275	* because we need to intercept it to prevent nested #DBs from hanging the
4276	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4277	*/
4278	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4279	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4280	&& !CPUMIsGuestDebugStateActive(pVCpu))
4281	#else
4282	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4283	#endif
4284	{
4285	fInterceptMovDRx = true;
4286	}
4287
4288	/* Update guest DR7. */
4289	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4290	AssertRCReturn(rc, rc);
4291
4292	pVCpu->hm.s.fUsingHyperDR7 = false;
4293	}
4294
4295	/*
4296	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4297	*/
4298	if (fInterceptMovDRx)
4299	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4300	else
4301	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4302	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4303	AssertRCReturn(rc, rc);
4304
4305	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4306	return VINF_SUCCESS;
4307	}
4308
4309
4310	#ifdef VBOX_STRICT
4311	/**
4312	* Strict function to validate segment registers.
4313	*
4314	* @remarks ASSUMES CR0 is up to date.
4315	*/
4316	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4317	{
4318	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4319	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4320	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4321	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4322	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4323	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4324	{
4325	/* Protected mode checks */
4326	/* CS */
4327	Assert(pCtx->cs.Attr.n.u1Present);
4328	Assert(!(pCtx->cs.Attr.u & 0xf00));
4329	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4330	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4331	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4332	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4333	\|\| (pCtx->cs.Attr.n.u1Granularity));
4334	/* CS cannot be loaded with NULL in protected mode. */
4335	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4336	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4337	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4338	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4339	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4340	else
4341	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4342	/* SS */
4343	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4344	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4345	if ( !(pCtx->cr0 & X86_CR0_PE)
4346	\|\| pCtx->cs.Attr.n.u4Type == 3)
4347	{
4348	Assert(!pCtx->ss.Attr.n.u2Dpl);
4349	}
4350	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4351	{
4352	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4353	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4354	Assert(pCtx->ss.Attr.n.u1Present);
4355	Assert(!(pCtx->ss.Attr.u & 0xf00));
4356	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4357	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4358	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4359	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4360	\|\| (pCtx->ss.Attr.n.u1Granularity));
4361	}
4362	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4363	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4364	{
4365	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4366	Assert(pCtx->ds.Attr.n.u1Present);
4367	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4368	Assert(!(pCtx->ds.Attr.u & 0xf00));
4369	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4370	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4371	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4372	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4373	\|\| (pCtx->ds.Attr.n.u1Granularity));
4374	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4375	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4376	}
4377	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4378	{
4379	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4380	Assert(pCtx->es.Attr.n.u1Present);
4381	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4382	Assert(!(pCtx->es.Attr.u & 0xf00));
4383	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4384	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4385	\|\| !(pCtx->es.Attr.n.u1Granularity));
4386	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4387	\|\| (pCtx->es.Attr.n.u1Granularity));
4388	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4389	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4390	}
4391	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4392	{
4393	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4394	Assert(pCtx->fs.Attr.n.u1Present);
4395	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4396	Assert(!(pCtx->fs.Attr.u & 0xf00));
4397	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4398	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4399	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4400	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4401	\|\| (pCtx->fs.Attr.n.u1Granularity));
4402	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4403	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4404	}
4405	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4406	{
4407	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4408	Assert(pCtx->gs.Attr.n.u1Present);
4409	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4410	Assert(!(pCtx->gs.Attr.u & 0xf00));
4411	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4412	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4413	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4414	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4415	\|\| (pCtx->gs.Attr.n.u1Granularity));
4416	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4417	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4418	}
4419	/* 64-bit capable CPUs. */
4420	# if HC_ARCH_BITS == 64
4421	Assert(!(pCtx->cs.u64Base >> 32));
4422	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4423	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4424	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4425	# endif
4426	}
4427	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4428	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4429	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4430	{
4431	/* Real and v86 mode checks. */
4432	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4433	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4434	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4435	{
4436	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4437	}
4438	else
4439	{
4440	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4441	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4442	}
4443
4444	/* CS */
4445	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4446	Assert(pCtx->cs.u32Limit == 0xffff);
4447	Assert(u32CSAttr == 0xf3);
4448	/* SS */
4449	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4450	Assert(pCtx->ss.u32Limit == 0xffff);
4451	Assert(u32SSAttr == 0xf3);
4452	/* DS */
4453	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4454	Assert(pCtx->ds.u32Limit == 0xffff);
4455	Assert(u32DSAttr == 0xf3);
4456	/* ES */
4457	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4458	Assert(pCtx->es.u32Limit == 0xffff);
4459	Assert(u32ESAttr == 0xf3);
4460	/* FS */
4461	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4462	Assert(pCtx->fs.u32Limit == 0xffff);
4463	Assert(u32FSAttr == 0xf3);
4464	/* GS */
4465	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4466	Assert(pCtx->gs.u32Limit == 0xffff);
4467	Assert(u32GSAttr == 0xf3);
4468	/* 64-bit capable CPUs. */
4469	# if HC_ARCH_BITS == 64
4470	Assert(!(pCtx->cs.u64Base >> 32));
4471	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4472	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4473	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4474	# endif
4475	}
4476	}
4477	#endif /* VBOX_STRICT */
4478
4479
4480	/**
4481	* Writes a guest segment register into the guest-state area in the VMCS.
4482	*
4483	* @returns VBox status code.
4484	* @param pVCpu The cross context virtual CPU structure.
4485	* @param idxSel Index of the selector in the VMCS.
4486	* @param idxLimit Index of the segment limit in the VMCS.
4487	* @param idxBase Index of the segment base in the VMCS.
4488	* @param idxAccess Index of the access rights of the segment in the VMCS.
4489	* @param pSelReg Pointer to the segment selector.
4490	*
4491	* @remarks No-long-jump zone!!!
4492	*/
4493	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4494	uint32_t idxAccess, PCPUMSELREG pSelReg)
4495	{
4496	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4497	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4498	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4499	AssertRCReturn(rc, rc);
4500
4501	uint32_t u32Access = pSelReg->Attr.u;
4502	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4503	{
4504	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4505	u32Access = 0xf3;
4506	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4507	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4508	}
4509	else
4510	{
4511	/*
4512	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4513	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4514	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4515	* loaded in protected-mode have their attribute as 0.
4516	*/
4517	if (!u32Access)
4518	u32Access = X86DESCATTR_UNUSABLE;
4519	}
4520
4521	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4522	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4523	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4524
4525	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4526	AssertRCReturn(rc, rc);
4527	return rc;
4528	}
4529
4530
4531	/**
4532	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4533	* into the guest-state area in the VMCS.
4534	*
4535	* @returns VBox status code.
4536	* @param pVCpu The cross context virtual CPU structure.
4537	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4538	* out-of-sync. Make sure to update the required fields
4539	* before using them.
4540	*
4541	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4542	* @remarks No-long-jump zone!!!
4543	*/
4544	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4545	{
4546	int rc = VERR_INTERNAL_ERROR_5;
4547	PVM pVM = pVCpu->CTX_SUFF(pVM);
4548
4549	/*
4550	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4551	*/
4552	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4553	{
4554	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4555	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4556	{
4557	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4558	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4559	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4560	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4561	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4562	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4563	}
4564
4565	#ifdef VBOX_WITH_REM
4566	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4567	{
4568	Assert(pVM->hm.s.vmx.pRealModeTSS);
4569	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4570	if ( pVCpu->hm.s.vmx.fWasInRealMode
4571	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4572	{
4573	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4574	in real-mode (e.g. OpenBSD 4.0) */
4575	REMFlushTBs(pVM);
4576	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4577	pVCpu->hm.s.vmx.fWasInRealMode = false;
4578	}
4579	}
4580	#endif
4581	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4582	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4583	AssertRCReturn(rc, rc);
4584	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4585	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4586	AssertRCReturn(rc, rc);
4587	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4588	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4589	AssertRCReturn(rc, rc);
4590	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4591	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4592	AssertRCReturn(rc, rc);
4593	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4594	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4595	AssertRCReturn(rc, rc);
4596	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4597	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4598	AssertRCReturn(rc, rc);
4599
4600	#ifdef VBOX_STRICT
4601	/* Validate. */
4602	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4603	#endif
4604
4605	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4606	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4607	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4608
4609	/* Update the exit history entry with the correct CS.BASE + RIP. */
4610	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
4611	EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4612	}
4613
4614	/*
4615	* Guest TR.
4616	*/
4617	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4618	{
4619	/*
4620	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4621	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4622	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4623	*/
4624	uint16_t u16Sel = 0;
4625	uint32_t u32Limit = 0;
4626	uint64_t u64Base = 0;
4627	uint32_t u32AccessRights = 0;
4628
4629	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4630	{
4631	u16Sel = pMixedCtx->tr.Sel;
4632	u32Limit = pMixedCtx->tr.u32Limit;
4633	u64Base = pMixedCtx->tr.u64Base;
4634	u32AccessRights = pMixedCtx->tr.Attr.u;
4635	}
4636	else
4637	{
4638	Assert(pVM->hm.s.vmx.pRealModeTSS);
4639	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4640
4641	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4642	RTGCPHYS GCPhys;
4643	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4644	AssertRCReturn(rc, rc);
4645
4646	X86DESCATTR DescAttr;
4647	DescAttr.u = 0;
4648	DescAttr.n.u1Present = 1;
4649	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4650
4651	u16Sel = 0;
4652	u32Limit = HM_VTX_TSS_SIZE;
4653	u64Base = GCPhys; /* in real-mode phys = virt. */
4654	u32AccessRights = DescAttr.u;
4655	}
4656
4657	/* Validate. */
4658	Assert(!(u16Sel & RT_BIT(2)));
4659	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4660	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4661	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4662	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4663	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4664	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4665	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4666	Assert( (u32Limit & 0xfff) == 0xfff
4667	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4668	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4669	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4670
4671	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4672	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4673	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4674	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4675	AssertRCReturn(rc, rc);
4676
4677	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4678	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4679	}
4680
4681	/*
4682	* Guest GDTR.
4683	*/
4684	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4685	{
4686	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4687	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4688	AssertRCReturn(rc, rc);
4689
4690	/* Validate. */
4691	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4692
4693	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4694	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4695	}
4696
4697	/*
4698	* Guest LDTR.
4699	*/
4700	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4701	{
4702	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4703	uint32_t u32Access = 0;
4704	if (!pMixedCtx->ldtr.Attr.u)
4705	u32Access = X86DESCATTR_UNUSABLE;
4706	else
4707	u32Access = pMixedCtx->ldtr.Attr.u;
4708
4709	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4710	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4711	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4712	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4713	AssertRCReturn(rc, rc);
4714
4715	/* Validate. */
4716	if (!(u32Access & X86DESCATTR_UNUSABLE))
4717	{
4718	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4719	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4720	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4721	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4722	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4723	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4724	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4725	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4726	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4727	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4728	}
4729
4730	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4731	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4732	}
4733
4734	/*
4735	* Guest IDTR.
4736	*/
4737	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4738	{
4739	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4740	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4741	AssertRCReturn(rc, rc);
4742
4743	/* Validate. */
4744	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4745
4746	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4747	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4748	}
4749
4750	return VINF_SUCCESS;
4751	}
4752
4753
4754	/**
4755	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4756	* areas.
4757	*
4758	* These MSRs will automatically be loaded to the host CPU on every successful
4759	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4760	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4761	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4762	*
4763	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4764	*
4765	* @returns VBox status code.
4766	* @param pVCpu The cross context virtual CPU structure.
4767	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4768	* out-of-sync. Make sure to update the required fields
4769	* before using them.
4770	*
4771	* @remarks No-long-jump zone!!!
4772	*/
4773	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4774	{
4775	AssertPtr(pVCpu);
4776	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4777
4778	/*
4779	* MSRs that we use the auto-load/store MSR area in the VMCS.
4780	*/
4781	PVM pVM = pVCpu->CTX_SUFF(pVM);
4782	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4783	{
4784	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4785	#if HC_ARCH_BITS == 32
4786	if (pVM->hm.s.fAllow64BitGuests)
4787	{
4788	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4789	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4790	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4791	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4792	AssertRCReturn(rc, rc);
4793	# ifdef LOG_ENABLED
4794	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4795	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4796	{
4797	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4798	pMsr->u64Value));
4799	}
4800	# endif
4801	}
4802	#endif
4803	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4804	}
4805
4806	/*
4807	* Guest Sysenter MSRs.
4808	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4809	* VM-exits on WRMSRs for these MSRs.
4810	*/
4811	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4812	{
4813	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4814	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4815	}
4816
4817	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4818	{
4819	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4820	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4821	}
4822
4823	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4824	{
4825	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4826	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4827	}
4828
4829	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4830	{
4831	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4832	{
4833	/*
4834	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4835	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4836	*/
4837	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4838	{
4839	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4840	AssertRCReturn(rc,rc);
4841	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4842	}
4843	else
4844	{
4845	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4846	NULL /* pfAddedAndUpdated */);
4847	AssertRCReturn(rc, rc);
4848
4849	/* We need to intercept reads too, see @bugref{7386#c16}. */
4850	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4851	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4852	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4853	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4854	}
4855	}
4856	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4857	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4858	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4859	}
4860
4861	return VINF_SUCCESS;
4862	}
4863
4864
4865	/**
4866	* Loads the guest activity state into the guest-state area in the VMCS.
4867	*
4868	* @returns VBox status code.
4869	* @param pVCpu The cross context virtual CPU structure.
4870	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4871	* out-of-sync. Make sure to update the required fields
4872	* before using them.
4873	*
4874	* @remarks No-long-jump zone!!!
4875	*/
4876	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4877	{
4878	NOREF(pMixedCtx);
4879	/** @todo See if we can make use of other states, e.g.
4880	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4881	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4882	{
4883	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4884	AssertRCReturn(rc, rc);
4885
4886	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4887	}
4888	return VINF_SUCCESS;
4889	}
4890
4891
4892	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4893	/**
4894	* Check if guest state allows safe use of 32-bit switcher again.
4895	*
4896	* Segment bases and protected mode structures must be 32-bit addressable
4897	* because the 32-bit switcher will ignore high dword when writing these VMCS
4898	* fields. See @bugref{8432} for details.
4899	*
4900	* @returns true if safe, false if must continue to use the 64-bit switcher.
4901	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4902	* out-of-sync. Make sure to update the required fields
4903	* before using them.
4904	*
4905	* @remarks No-long-jump zone!!!
4906	*/
4907	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4908	{
4909	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4910	return false;
4911	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4912	return false;
4913	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4914	return false;
4915	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4916	return false;
4917	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4918	return false;
4919	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4920	return false;
4921	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4922	return false;
4923	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4924	return false;
4925	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4926	return false;
4927	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4928	return false;
4929	/* All good, bases are 32-bit. */
4930	return true;
4931	}
4932	#endif
4933
4934
4935	/**
4936	* Sets up the appropriate function to run guest code.
4937	*
4938	* @returns VBox status code.
4939	* @param pVCpu The cross context virtual CPU structure.
4940	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4941	* out-of-sync. Make sure to update the required fields
4942	* before using them.
4943	*
4944	* @remarks No-long-jump zone!!!
4945	*/
4946	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4947	{
4948	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4949	{
4950	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4951	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4952	#endif
4953	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4954	#if HC_ARCH_BITS == 32
4955	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4956	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4957	{
4958	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4959	{
4960	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4961	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4962	\| HM_CHANGED_VMX_ENTRY_CTLS
4963	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4964	}
4965	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4966
4967	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4968	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4969	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4970	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4971	}
4972	#else
4973	/* 64-bit host. */
4974	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4975	#endif
4976	}
4977	else
4978	{
4979	/* Guest is not in long mode, use the 32-bit handler. */
4980	#if HC_ARCH_BITS == 32
4981	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4982	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4983	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4984	{
4985	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4986	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4987	\| HM_CHANGED_VMX_ENTRY_CTLS
4988	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4989	}
4990	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4991	/*
4992	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4993	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4994	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4995	* the much faster 32-bit switcher again.
4996	*/
4997	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4998	{
4999	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
5000	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
5001	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5002	}
5003	else
5004	{
5005	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5006	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5007	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5008	{
5009	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5010	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5011	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5012	\| HM_CHANGED_VMX_ENTRY_CTLS
5013	\| HM_CHANGED_VMX_EXIT_CTLS
5014	\| HM_CHANGED_HOST_CONTEXT);
5015	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5016	}
5017	}
5018	# else
5019	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5020	# endif
5021	#else
5022	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5023	#endif
5024	}
5025	Assert(pVCpu->hm.s.vmx.pfnStartVM);
5026	return VINF_SUCCESS;
5027	}
5028
5029
5030	/**
5031	* Wrapper for running the guest code in VT-x.
5032	*
5033	* @returns VBox status code, no informational status codes.
5034	* @param pVM The cross context VM structure.
5035	* @param pVCpu The cross context virtual CPU structure.
5036	* @param pCtx Pointer to the guest-CPU context.
5037	*
5038	* @remarks No-long-jump zone!!!
5039	*/
5040	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5041	{
5042	/*
5043	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5044	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5045	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5046	*/
5047	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5048	/** @todo Add stats for resume vs launch. */
5049	#ifdef VBOX_WITH_KERNEL_USING_XMM
5050	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5051	#else
5052	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5053	#endif
5054	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5055	return rc;
5056	}
5057
5058
5059	/**
5060	* Reports world-switch error and dumps some useful debug info.
5061	*
5062	* @param pVM The cross context VM structure.
5063	* @param pVCpu The cross context virtual CPU structure.
5064	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5065	* @param pCtx Pointer to the guest-CPU context.
5066	* @param pVmxTransient Pointer to the VMX transient structure (only
5067	* exitReason updated).
5068	*/
5069	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5070	{
5071	Assert(pVM);
5072	Assert(pVCpu);
5073	Assert(pCtx);
5074	Assert(pVmxTransient);
5075	HMVMX_ASSERT_PREEMPT_SAFE();
5076
5077	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5078	switch (rcVMRun)
5079	{
5080	case VERR_VMX_INVALID_VMXON_PTR:
5081	AssertFailed();
5082	break;
5083	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5084	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5085	{
5086	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5087	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5088	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5089	AssertRC(rc);
5090
5091	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5092	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5093	Cannot do it here as we may have been long preempted. */
5094
5095	#ifdef VBOX_STRICT
5096	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5097	pVmxTransient->uExitReason));
5098	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5099	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5100	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5101	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5102	else
5103	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5104	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5105	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5106
5107	/* VMX control bits. */
5108	uint32_t u32Val;
5109	uint64_t u64Val;
5110	RTHCUINTREG uHCReg;
5111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5112	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5114	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5115	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5116	{
5117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5118	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5119	}
5120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5121	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5123	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5125	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5127	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5128	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5129	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5131	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5132	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5133	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5135	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5136	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5137	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5138	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5139	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5140	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5141	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5142	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5143	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5144	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5145	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5146	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5147	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5148	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5149	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5150	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5151	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5152	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5153	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5154	if (pVM->hm.s.fNestedPaging)
5155	{
5156	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5157	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5158	}
5159
5160	/* Guest bits. */
5161	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5162	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5163	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5164	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5165	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5166	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5167	if (pVM->hm.s.vmx.fVpid)
5168	{
5169	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5170	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5171	}
5172
5173	/* Host bits. */
5174	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5175	Log4(("Host CR0 %#RHr\n", uHCReg));
5176	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5177	Log4(("Host CR3 %#RHr\n", uHCReg));
5178	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5179	Log4(("Host CR4 %#RHr\n", uHCReg));
5180
5181	RTGDTR HostGdtr;
5182	PCX86DESCHC pDesc;
5183	ASMGetGDTR(&HostGdtr);
5184	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5185	Log4(("Host CS %#08x\n", u32Val));
5186	if (u32Val < HostGdtr.cbGdt)
5187	{
5188	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5189	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5190	}
5191
5192	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5193	Log4(("Host DS %#08x\n", u32Val));
5194	if (u32Val < HostGdtr.cbGdt)
5195	{
5196	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5197	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5198	}
5199
5200	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5201	Log4(("Host ES %#08x\n", u32Val));
5202	if (u32Val < HostGdtr.cbGdt)
5203	{
5204	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5205	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5206	}
5207
5208	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5209	Log4(("Host FS %#08x\n", u32Val));
5210	if (u32Val < HostGdtr.cbGdt)
5211	{
5212	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5213	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5214	}
5215
5216	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5217	Log4(("Host GS %#08x\n", u32Val));
5218	if (u32Val < HostGdtr.cbGdt)
5219	{
5220	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5221	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5222	}
5223
5224	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5225	Log4(("Host SS %#08x\n", u32Val));
5226	if (u32Val < HostGdtr.cbGdt)
5227	{
5228	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5229	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5230	}
5231
5232	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5233	Log4(("Host TR %#08x\n", u32Val));
5234	if (u32Val < HostGdtr.cbGdt)
5235	{
5236	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5237	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5238	}
5239
5240	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5241	Log4(("Host TR Base %#RHv\n", uHCReg));
5242	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5243	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5244	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5245	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5246	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5247	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5248	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5249	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5250	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5251	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5252	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5253	Log4(("Host RSP %#RHv\n", uHCReg));
5254	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5255	Log4(("Host RIP %#RHv\n", uHCReg));
5256	# if HC_ARCH_BITS == 64
5257	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5258	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5259	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5260	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5261	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5262	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5263	# endif
5264	#endif /* VBOX_STRICT */
5265	break;
5266	}
5267
5268	default:
5269	/* Impossible */
5270	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5271	break;
5272	}
5273	NOREF(pVM); NOREF(pCtx);
5274	}
5275
5276
5277	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5278	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5279	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5280	#endif
5281	#ifdef VBOX_STRICT
5282	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5283	{
5284	switch (idxField)
5285	{
5286	case VMX_VMCS_GUEST_RIP:
5287	case VMX_VMCS_GUEST_RSP:
5288	case VMX_VMCS_GUEST_SYSENTER_EIP:
5289	case VMX_VMCS_GUEST_SYSENTER_ESP:
5290	case VMX_VMCS_GUEST_GDTR_BASE:
5291	case VMX_VMCS_GUEST_IDTR_BASE:
5292	case VMX_VMCS_GUEST_CS_BASE:
5293	case VMX_VMCS_GUEST_DS_BASE:
5294	case VMX_VMCS_GUEST_ES_BASE:
5295	case VMX_VMCS_GUEST_FS_BASE:
5296	case VMX_VMCS_GUEST_GS_BASE:
5297	case VMX_VMCS_GUEST_SS_BASE:
5298	case VMX_VMCS_GUEST_LDTR_BASE:
5299	case VMX_VMCS_GUEST_TR_BASE:
5300	case VMX_VMCS_GUEST_CR3:
5301	return true;
5302	}
5303	return false;
5304	}
5305
5306	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5307	{
5308	switch (idxField)
5309	{
5310	/* Read-only fields. */
5311	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5312	return true;
5313	}
5314	/* Remaining readable fields should also be writable. */
5315	return hmR0VmxIsValidWriteField(idxField);
5316	}
5317	#endif /* VBOX_STRICT */
5318
5319
5320	/**
5321	* Executes the specified handler in 64-bit mode.
5322	*
5323	* @returns VBox status code (no informational status codes).
5324	* @param pVM The cross context VM structure.
5325	* @param pVCpu The cross context virtual CPU structure.
5326	* @param pCtx Pointer to the guest CPU context.
5327	* @param enmOp The operation to perform.
5328	* @param cParams Number of parameters.
5329	* @param paParam Array of 32-bit parameters.
5330	*/
5331	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5332	uint32_t cParams, uint32_t *paParam)
5333	{
5334	NOREF(pCtx);
5335
5336	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5337	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5338	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5339	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5340
5341	#ifdef VBOX_STRICT
5342	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5343	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5344
5345	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5346	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5347	#endif
5348
5349	/* Disable interrupts. */
5350	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5351
5352	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5353	RTCPUID idHostCpu = RTMpCpuId();
5354	CPUMR0SetLApic(pVCpu, idHostCpu);
5355	#endif
5356
5357	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5358	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5359
5360	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5361	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5362	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5363
5364	/* Leave VMX Root Mode. */
5365	VMXDisable();
5366
5367	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5368
5369	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5370	CPUMSetHyperEIP(pVCpu, enmOp);
5371	for (int i = (int)cParams - 1; i >= 0; i--)
5372	CPUMPushHyper(pVCpu, paParam[i]);
5373
5374	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5375
5376	/* Call the switcher. */
5377	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5378	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5379
5380	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5381	/* Make sure the VMX instructions don't cause #UD faults. */
5382	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5383
5384	/* Re-enter VMX Root Mode */
5385	int rc2 = VMXEnable(HCPhysCpuPage);
5386	if (RT_FAILURE(rc2))
5387	{
5388	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5389	ASMSetFlags(fOldEFlags);
5390	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5391	return rc2;
5392	}
5393
5394	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5395	AssertRC(rc2);
5396	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5397	Assert(!(ASMGetFlags() & X86_EFL_IF));
5398	ASMSetFlags(fOldEFlags);
5399	return rc;
5400	}
5401
5402
5403	/**
5404	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5405	* supporting 64-bit guests.
5406	*
5407	* @returns VBox status code.
5408	* @param fResume Whether to VMLAUNCH or VMRESUME.
5409	* @param pCtx Pointer to the guest-CPU context.
5410	* @param pCache Pointer to the VMCS cache.
5411	* @param pVM The cross context VM structure.
5412	* @param pVCpu The cross context virtual CPU structure.
5413	*/
5414	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5415	{
5416	NOREF(fResume);
5417
5418	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5419	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5420
5421	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5422	pCache->uPos = 1;
5423	pCache->interPD = PGMGetInterPaeCR3(pVM);
5424	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5425	#endif
5426
5427	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5428	pCache->TestIn.HCPhysCpuPage = 0;
5429	pCache->TestIn.HCPhysVmcs = 0;
5430	pCache->TestIn.pCache = 0;
5431	pCache->TestOut.HCPhysVmcs = 0;
5432	pCache->TestOut.pCache = 0;
5433	pCache->TestOut.pCtx = 0;
5434	pCache->TestOut.eflags = 0;
5435	#else
5436	NOREF(pCache);
5437	#endif
5438
5439	uint32_t aParam[10];
5440	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5441	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5442	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5443	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5444	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5445	aParam[5] = 0;
5446	aParam[6] = VM_RC_ADDR(pVM, pVM);
5447	aParam[7] = 0;
5448	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5449	aParam[9] = 0;
5450
5451	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5452	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5453	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5454	#endif
5455	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5456
5457	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5458	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5459	Assert(pCtx->dr[4] == 10);
5460	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5461	#endif
5462
5463	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5464	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5465	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5466	pVCpu->hm.s.vmx.HCPhysVmcs));
5467	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5468	pCache->TestOut.HCPhysVmcs));
5469	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5470	pCache->TestOut.pCache));
5471	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5472	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5473	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5474	pCache->TestOut.pCtx));
5475	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5476	#endif
5477	return rc;
5478	}
5479
5480
5481	/**
5482	* Initialize the VMCS-Read cache.
5483	*
5484	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5485	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5486	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5487	* (those that have a 32-bit FULL & HIGH part).
5488	*
5489	* @returns VBox status code.
5490	* @param pVM The cross context VM structure.
5491	* @param pVCpu The cross context virtual CPU structure.
5492	*/
5493	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5494	{
5495	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5496	{ \
5497	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5498	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5499	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5500	++cReadFields; \
5501	}
5502
5503	AssertPtr(pVM);
5504	AssertPtr(pVCpu);
5505	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5506	uint32_t cReadFields = 0;
5507
5508	/*
5509	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5510	* and serve to indicate exceptions to the rules.
5511	*/
5512
5513	/* Guest-natural selector base fields. */
5514	#if 0
5515	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5518	#endif
5519	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5520	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5530	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5531	#if 0
5532	/* Unused natural width guest-state fields. */
5533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5534	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5535	#endif
5536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5537	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5538
5539	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5540	#if 0
5541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5542	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5543	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5544	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5545	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5546	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5547	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5548	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5549	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5550	#endif
5551
5552	/* Natural width guest-state fields. */
5553	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5554	#if 0
5555	/* Currently unused field. */
5556	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5557	#endif
5558
5559	if (pVM->hm.s.fNestedPaging)
5560	{
5561	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5562	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5563	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5564	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5565	}
5566	else
5567	{
5568	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5569	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5570	}
5571
5572	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5573	return VINF_SUCCESS;
5574	}
5575
5576
5577	/**
5578	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5579	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5580	* darwin, running 64-bit guests).
5581	*
5582	* @returns VBox status code.
5583	* @param pVCpu The cross context virtual CPU structure.
5584	* @param idxField The VMCS field encoding.
5585	* @param u64Val 16, 32 or 64-bit value.
5586	*/
5587	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5588	{
5589	int rc;
5590	switch (idxField)
5591	{
5592	/*
5593	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5594	*/
5595	/* 64-bit Control fields. */
5596	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5597	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5598	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5599	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5600	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5601	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5602	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5603	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5604	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5605	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5606	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5607	case VMX_VMCS64_CTRL_EPTP_FULL:
5608	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5609	/* 64-bit Guest-state fields. */
5610	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5611	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5612	case VMX_VMCS64_GUEST_PAT_FULL:
5613	case VMX_VMCS64_GUEST_EFER_FULL:
5614	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5615	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5616	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5617	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5618	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5619	/* 64-bit Host-state fields. */
5620	case VMX_VMCS64_HOST_PAT_FULL:
5621	case VMX_VMCS64_HOST_EFER_FULL:
5622	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5623	{
5624	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5625	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5626	break;
5627	}
5628
5629	/*
5630	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5631	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5632	*/
5633	/* Natural-width Guest-state fields. */
5634	case VMX_VMCS_GUEST_CR3:
5635	case VMX_VMCS_GUEST_ES_BASE:
5636	case VMX_VMCS_GUEST_CS_BASE:
5637	case VMX_VMCS_GUEST_SS_BASE:
5638	case VMX_VMCS_GUEST_DS_BASE:
5639	case VMX_VMCS_GUEST_FS_BASE:
5640	case VMX_VMCS_GUEST_GS_BASE:
5641	case VMX_VMCS_GUEST_LDTR_BASE:
5642	case VMX_VMCS_GUEST_TR_BASE:
5643	case VMX_VMCS_GUEST_GDTR_BASE:
5644	case VMX_VMCS_GUEST_IDTR_BASE:
5645	case VMX_VMCS_GUEST_RSP:
5646	case VMX_VMCS_GUEST_RIP:
5647	case VMX_VMCS_GUEST_SYSENTER_ESP:
5648	case VMX_VMCS_GUEST_SYSENTER_EIP:
5649	{
5650	if (!(RT_HI_U32(u64Val)))
5651	{
5652	/* If this field is 64-bit, VT-x will zero out the top bits. */
5653	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5654	}
5655	else
5656	{
5657	/* Assert that only the 32->64 switcher case should ever come here. */
5658	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5659	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5660	}
5661	break;
5662	}
5663
5664	default:
5665	{
5666	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5667	rc = VERR_INVALID_PARAMETER;
5668	break;
5669	}
5670	}
5671	AssertRCReturn(rc, rc);
5672	return rc;
5673	}
5674
5675
5676	/**
5677	* Queue up a VMWRITE by using the VMCS write cache.
5678	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5679	*
5680	* @param pVCpu The cross context virtual CPU structure.
5681	* @param idxField The VMCS field encoding.
5682	* @param u64Val 16, 32 or 64-bit value.
5683	*/
5684	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5685	{
5686	AssertPtr(pVCpu);
5687	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5688
5689	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5690	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5691
5692	/* Make sure there are no duplicates. */
5693	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5694	{
5695	if (pCache->Write.aField[i] == idxField)
5696	{
5697	pCache->Write.aFieldVal[i] = u64Val;
5698	return VINF_SUCCESS;
5699	}
5700	}
5701
5702	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5703	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5704	pCache->Write.cValidEntries++;
5705	return VINF_SUCCESS;
5706	}
5707	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5708
5709
5710	/**
5711	* Sets up the usage of TSC-offsetting and updates the VMCS.
5712	*
5713	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5714	* VMX preemption timer.
5715	*
5716	* @returns VBox status code.
5717	* @param pVM The cross context VM structure.
5718	* @param pVCpu The cross context virtual CPU structure.
5719	*
5720	* @remarks No-long-jump zone!!!
5721	*/
5722	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5723	{
5724	int rc;
5725	bool fOffsettedTsc;
5726	bool fParavirtTsc;
5727	if (pVM->hm.s.vmx.fUsePreemptTimer)
5728	{
5729	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5730	&fOffsettedTsc, &fParavirtTsc);
5731
5732	/* Make sure the returned values have sane upper and lower boundaries. */
5733	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5734	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5735	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5736	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5737
5738	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5739	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5740	}
5741	else
5742	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5743
5744	/** @todo later optimize this to be done elsewhere and not before every
5745	* VM-entry. */
5746	if (fParavirtTsc)
5747	{
5748	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5749	information before every VM-entry, hence disable it for performance sake. */
5750	#if 0
5751	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5752	AssertRC(rc);
5753	#endif
5754	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5755	}
5756
5757	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5758	{
5759	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5760	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5761
5762	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5763	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5764	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5765	}
5766	else
5767	{
5768	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5769	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5770	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5771	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5772	}
5773	}
5774
5775
5776	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5777	/**
5778	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5779	* VM-exit interruption info type.
5780	*
5781	* @returns The IEM exception flags.
5782	* @param uVector The event vector.
5783	* @param uVmxVectorType The VMX event type.
5784	*
5785	* @remarks This function currently only constructs flags required for
5786	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5787	* and CR2 aspects of an exception are not included).
5788	*/
5789	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5790	{
5791	uint32_t fIemXcptFlags;
5792	switch (uVmxVectorType)
5793	{
5794	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5795	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5796	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5797	break;
5798
5799	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5800	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5801	break;
5802
5803	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5804	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5805	break;
5806
5807	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5808	{
5809	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5810	if (uVector == X86_XCPT_BP)
5811	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5812	else if (uVector == X86_XCPT_OF)
5813	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5814	else
5815	{
5816	fIemXcptFlags = 0;
5817	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5818	}
5819	break;
5820	}
5821
5822	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5823	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5824	break;
5825
5826	default:
5827	fIemXcptFlags = 0;
5828	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5829	break;
5830	}
5831	return fIemXcptFlags;
5832	}
5833
5834	#else
5835	/**
5836	* Determines if an exception is a contributory exception.
5837	*
5838	* Contributory exceptions are ones which can cause double-faults unless the
5839	* original exception was a benign exception. Page-fault is intentionally not
5840	* included here as it's a conditional contributory exception.
5841	*
5842	* @returns true if the exception is contributory, false otherwise.
5843	* @param uVector The exception vector.
5844	*/
5845	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5846	{
5847	switch (uVector)
5848	{
5849	case X86_XCPT_GP:
5850	case X86_XCPT_SS:
5851	case X86_XCPT_NP:
5852	case X86_XCPT_TS:
5853	case X86_XCPT_DE:
5854	return true;
5855	default:
5856	break;
5857	}
5858	return false;
5859	}
5860	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5861
5862
5863	/**
5864	* Sets an event as a pending event to be injected into the guest.
5865	*
5866	* @param pVCpu The cross context virtual CPU structure.
5867	* @param u32IntInfo The VM-entry interruption-information field.
5868	* @param cbInstr The VM-entry instruction length in bytes (for software
5869	* interrupts, exceptions and privileged software
5870	* exceptions).
5871	* @param u32ErrCode The VM-entry exception error code.
5872	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5873	* page-fault.
5874	*
5875	* @remarks Statistics counter assumes this is a guest event being injected or
5876	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5877	* always incremented.
5878	*/
5879	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5880	RTGCUINTPTR GCPtrFaultAddress)
5881	{
5882	Assert(!pVCpu->hm.s.Event.fPending);
5883	pVCpu->hm.s.Event.fPending = true;
5884	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5885	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5886	pVCpu->hm.s.Event.cbInstr = cbInstr;
5887	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5888	}
5889
5890
5891	/**
5892	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5893	*
5894	* @param pVCpu The cross context virtual CPU structure.
5895	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5896	* out-of-sync. Make sure to update the required fields
5897	* before using them.
5898	*/
5899	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5900	{
5901	NOREF(pMixedCtx);
5902	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5903	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5904	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5905	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5906	}
5907
5908
5909	/**
5910	* Handle a condition that occurred while delivering an event through the guest
5911	* IDT.
5912	*
5913	* @returns Strict VBox status code (i.e. informational status codes too).
5914	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5915	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5916	* to continue execution of the guest which will delivery the \#DF.
5917	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5918	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5919	*
5920	* @param pVCpu The cross context virtual CPU structure.
5921	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5922	* out-of-sync. Make sure to update the required fields
5923	* before using them.
5924	* @param pVmxTransient Pointer to the VMX transient structure.
5925	*
5926	* @remarks No-long-jump zone!!!
5927	*/
5928	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5929	{
5930	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5931
5932	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5933	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5934
5935	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5936	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5937	{
5938	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5939	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5940	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5941	/*
5942	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5943	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5944	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5945	* exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5946	* currently has no way of storing that these exceptions were caused by these instructions
5947	* (ICEBP's #DB poses the problem).
5948	*/
5949	IEMXCPTRAISE enmRaise;
5950	IEMXCPTRAISEINFO fRaiseInfo;
5951	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5952	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5953	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5954	{
5955	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5956	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5957	}
5958	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5959	{
5960	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5961	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5962	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5963	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5964	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5965	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5966	uExitVectorType), VERR_VMX_IPE_5);
5967	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5968
5969	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5970	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5971	{
5972	pVmxTransient->fVectoringPF = true;
5973	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5974	}
5975	}
5976	else
5977	{
5978	/*
5979	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5980	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5981	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5982	*/
5983	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5984	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5985	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5986	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5987	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5988	}
5989
5990	/*
5991	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5992	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5993	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5994	* subsequent VM-entry would fail.
5995	*
5996	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5997	*/
5998	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5999	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6000	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6001	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6002	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6003	{
6004	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6005	}
6006
6007	switch (enmRaise)
6008	{
6009	case IEMXCPTRAISE_CURRENT_XCPT:
6010	{
6011	Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6012	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6013	Assert(rcStrict == VINF_SUCCESS);
6014	break;
6015	}
6016
6017	case IEMXCPTRAISE_PREV_EVENT:
6018	{
6019	uint32_t u32ErrCode;
6020	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6021	{
6022	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6023	AssertRCReturn(rc2, rc2);
6024	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6025	}
6026	else
6027	u32ErrCode = 0;
6028
6029	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6030	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6031	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6032	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6033
6034	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6035	pVCpu->hm.s.Event.u32ErrCode));
6036	Assert(rcStrict == VINF_SUCCESS);
6037	break;
6038	}
6039
6040	case IEMXCPTRAISE_REEXEC_INSTR:
6041	Assert(rcStrict == VINF_SUCCESS);
6042	break;
6043
6044	case IEMXCPTRAISE_DOUBLE_FAULT:
6045	{
6046	/*
6047	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6048	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6049	*/
6050	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6051	{
6052	pVmxTransient->fVectoringDoublePF = true;
6053	Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6054	pMixedCtx->cr2));
6055	rcStrict = VINF_SUCCESS;
6056	}
6057	else
6058	{
6059	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6060	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6061	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6062	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6063	rcStrict = VINF_HM_DOUBLE_FAULT;
6064	}
6065	break;
6066	}
6067
6068	case IEMXCPTRAISE_TRIPLE_FAULT:
6069	{
6070	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6071	uExitVector));
6072	rcStrict = VINF_EM_RESET;
6073	break;
6074	}
6075
6076	case IEMXCPTRAISE_CPU_HANG:
6077	{
6078	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6079	rcStrict = VERR_EM_GUEST_CPU_HANG;
6080	break;
6081	}
6082
6083	default:
6084	{
6085	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6086	rcStrict = VERR_VMX_IPE_2;
6087	break;
6088	}
6089	}
6090	#else
6091	typedef enum
6092	{
6093	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6094	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6095	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6096	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6097	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6098	} VMXREFLECTXCPT;
6099
6100	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6101	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6102	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6103	{
6104	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6105	{
6106	enmReflect = VMXREFLECTXCPT_XCPT;
6107	#ifdef VBOX_STRICT
6108	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6109	&& uExitVector == X86_XCPT_PF)
6110	{
6111	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6112	}
6113	#endif
6114	if ( uExitVector == X86_XCPT_PF
6115	&& uIdtVector == X86_XCPT_PF)
6116	{
6117	pVmxTransient->fVectoringDoublePF = true;
6118	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6119	}
6120	else if ( uExitVector == X86_XCPT_AC
6121	&& uIdtVector == X86_XCPT_AC)
6122	{
6123	enmReflect = VMXREFLECTXCPT_HANG;
6124	Log4(("IDT: Nested #AC - Bad guest\n"));
6125	}
6126	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6127	&& hmR0VmxIsContributoryXcpt(uExitVector)
6128	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6129	\|\| uIdtVector == X86_XCPT_PF))
6130	{
6131	enmReflect = VMXREFLECTXCPT_DF;
6132	}
6133	else if (uIdtVector == X86_XCPT_DF)
6134	enmReflect = VMXREFLECTXCPT_TF;
6135	}
6136	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6137	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6138	{
6139	/*
6140	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6141	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6142	*/
6143	enmReflect = VMXREFLECTXCPT_XCPT;
6144
6145	if (uExitVector == X86_XCPT_PF)
6146	{
6147	pVmxTransient->fVectoringPF = true;
6148	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6149	}
6150	}
6151	}
6152	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6153	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6154	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6155	{
6156	/*
6157	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6158	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6159	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6160	*/
6161	enmReflect = VMXREFLECTXCPT_XCPT;
6162	}
6163
6164	/*
6165	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6166	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6167	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6168	*
6169	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6170	*/
6171	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6172	&& enmReflect == VMXREFLECTXCPT_XCPT
6173	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6174	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6175	{
6176	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6177	}
6178
6179	switch (enmReflect)
6180	{
6181	case VMXREFLECTXCPT_XCPT:
6182	{
6183	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6184	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6185	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6186
6187	uint32_t u32ErrCode = 0;
6188	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6189	{
6190	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6191	AssertRCReturn(rc2, rc2);
6192	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6193	}
6194
6195	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6196	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6197	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6198	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6199	rcStrict = VINF_SUCCESS;
6200	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6201	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6202
6203	break;
6204	}
6205
6206	case VMXREFLECTXCPT_DF:
6207	{
6208	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6209	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6210	rcStrict = VINF_HM_DOUBLE_FAULT;
6211	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6212	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6213
6214	break;
6215	}
6216
6217	case VMXREFLECTXCPT_TF:
6218	{
6219	rcStrict = VINF_EM_RESET;
6220	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6221	uExitVector));
6222	break;
6223	}
6224
6225	case VMXREFLECTXCPT_HANG:
6226	{
6227	rcStrict = VERR_EM_GUEST_CPU_HANG;
6228	break;
6229	}
6230
6231	default:
6232	Assert(rcStrict == VINF_SUCCESS);
6233	break;
6234	}
6235	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6236	}
6237	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6238	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6239	&& uExitVector != X86_XCPT_DF
6240	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6241	{
6242	/*
6243	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6244	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6245	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6246	*/
6247	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6248	{
6249	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6250	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6251	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6252	}
6253	}
6254
6255	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6256	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6257	return rcStrict;
6258	}
6259
6260
6261	/**
6262	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6263	*
6264	* @returns VBox status code.
6265	* @param pVCpu The cross context virtual CPU structure.
6266	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6267	* out-of-sync. Make sure to update the required fields
6268	* before using them.
6269	*
6270	* @remarks No-long-jump zone!!!
6271	*/
6272	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6273	{
6274	NOREF(pMixedCtx);
6275
6276	/*
6277	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6278	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6279	*/
6280	VMMRZCallRing3Disable(pVCpu);
6281	HM_DISABLE_PREEMPT();
6282
6283	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6284	{
6285	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6286	* and 'dbgc-init' containing:
6287	* sxe "xcpt_de"
6288	* sxe "xcpt_bp"
6289	* sxi "xcpt_gp"
6290	* sxi "xcpt_ss"
6291	* sxi "xcpt_np"
6292	*/
6293	/** @todo r=ramshankar: Should be fixed after r119291. */
6294	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6295	#endif
6296	uint32_t uVal = 0;
6297	uint32_t uShadow = 0;
6298	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6299	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6300	AssertRCReturn(rc, rc);
6301
6302	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6303	CPUMSetGuestCR0(pVCpu, uVal);
6304	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6305	}
6306
6307	HM_RESTORE_PREEMPT();
6308	VMMRZCallRing3Enable(pVCpu);
6309	return VINF_SUCCESS;
6310	}
6311
6312
6313	/**
6314	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6315	*
6316	* @returns VBox status code.
6317	* @param pVCpu The cross context virtual CPU structure.
6318	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6319	* out-of-sync. Make sure to update the required fields
6320	* before using them.
6321	*
6322	* @remarks No-long-jump zone!!!
6323	*/
6324	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6325	{
6326	NOREF(pMixedCtx);
6327
6328	int rc = VINF_SUCCESS;
6329	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6330	{
6331	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6332	uint32_t uVal = 0;
6333	uint32_t uShadow = 0;
6334	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6335	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6336	AssertRCReturn(rc, rc);
6337
6338	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6339	CPUMSetGuestCR4(pVCpu, uVal);
6340	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6341	}
6342	return rc;
6343	}
6344
6345
6346	/**
6347	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6348	*
6349	* @returns VBox status code.
6350	* @param pVCpu The cross context virtual CPU structure.
6351	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6352	* out-of-sync. Make sure to update the required fields
6353	* before using them.
6354	*
6355	* @remarks No-long-jump zone!!!
6356	*/
6357	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6358	{
6359	int rc = VINF_SUCCESS;
6360	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6361	{
6362	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6363	uint64_t u64Val = 0;
6364	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6365	AssertRCReturn(rc, rc);
6366
6367	pMixedCtx->rip = u64Val;
6368	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6369	}
6370	return rc;
6371	}
6372
6373
6374	/**
6375	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6376	*
6377	* @returns VBox status code.
6378	* @param pVCpu The cross context virtual CPU structure.
6379	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6380	* out-of-sync. Make sure to update the required fields
6381	* before using them.
6382	*
6383	* @remarks No-long-jump zone!!!
6384	*/
6385	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6386	{
6387	int rc = VINF_SUCCESS;
6388	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6389	{
6390	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6391	uint64_t u64Val = 0;
6392	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6393	AssertRCReturn(rc, rc);
6394
6395	pMixedCtx->rsp = u64Val;
6396	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6397	}
6398	return rc;
6399	}
6400
6401
6402	/**
6403	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6404	*
6405	* @returns VBox status code.
6406	* @param pVCpu The cross context virtual CPU structure.
6407	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6408	* out-of-sync. Make sure to update the required fields
6409	* before using them.
6410	*
6411	* @remarks No-long-jump zone!!!
6412	*/
6413	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6414	{
6415	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6416	{
6417	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6418	uint32_t uVal = 0;
6419	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6420	AssertRCReturn(rc, rc);
6421
6422	pMixedCtx->eflags.u32 = uVal;
6423	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6424	{
6425	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6426	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6427
6428	pMixedCtx->eflags.Bits.u1VM = 0;
6429	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6430	}
6431
6432	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6433	}
6434	return VINF_SUCCESS;
6435	}
6436
6437
6438	/**
6439	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6440	* guest-CPU context.
6441	*/
6442	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6443	{
6444	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6445	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6446	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6447	return rc;
6448	}
6449
6450
6451	/**
6452	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6453	* from the guest-state area in the VMCS.
6454	*
6455	* @param pVCpu The cross context virtual CPU structure.
6456	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6457	* out-of-sync. Make sure to update the required fields
6458	* before using them.
6459	*
6460	* @remarks No-long-jump zone!!!
6461	*/
6462	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6463	{
6464	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6465	{
6466	uint32_t uIntrState = 0;
6467	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6468	AssertRC(rc);
6469
6470	if (!uIntrState)
6471	{
6472	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6473	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6474
6475	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6476	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6477	}
6478	else
6479	{
6480	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6481	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6482	{
6483	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6484	AssertRC(rc);
6485	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6486	AssertRC(rc);
6487
6488	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6489	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6490	}
6491	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6492	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6493
6494	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6495	{
6496	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6497	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6498	}
6499	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6500	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6501	}
6502
6503	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6504	}
6505	}
6506
6507
6508	/**
6509	* Saves the guest's activity state.
6510	*
6511	* @returns VBox status code.
6512	* @param pVCpu The cross context virtual CPU structure.
6513	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6514	* out-of-sync. Make sure to update the required fields
6515	* before using them.
6516	*
6517	* @remarks No-long-jump zone!!!
6518	*/
6519	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6520	{
6521	NOREF(pMixedCtx);
6522	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6523	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6524	return VINF_SUCCESS;
6525	}
6526
6527
6528	/**
6529	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6530	* the current VMCS into the guest-CPU context.
6531	*
6532	* @returns VBox status code.
6533	* @param pVCpu The cross context virtual CPU structure.
6534	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6535	* out-of-sync. Make sure to update the required fields
6536	* before using them.
6537	*
6538	* @remarks No-long-jump zone!!!
6539	*/
6540	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6541	{
6542	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6543	{
6544	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6545	uint32_t u32Val = 0;
6546	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6547	pMixedCtx->SysEnter.cs = u32Val;
6548	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6549	}
6550
6551	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6552	{
6553	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6554	uint64_t u64Val = 0;
6555	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6556	pMixedCtx->SysEnter.eip = u64Val;
6557	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6558	}
6559	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6560	{
6561	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6562	uint64_t u64Val = 0;
6563	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6564	pMixedCtx->SysEnter.esp = u64Val;
6565	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6566	}
6567	return VINF_SUCCESS;
6568	}
6569
6570
6571	/**
6572	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6573	* the CPU back into the guest-CPU context.
6574	*
6575	* @returns VBox status code.
6576	* @param pVCpu The cross context virtual CPU structure.
6577	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6578	* out-of-sync. Make sure to update the required fields
6579	* before using them.
6580	*
6581	* @remarks No-long-jump zone!!!
6582	*/
6583	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6584	{
6585	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6586	VMMRZCallRing3Disable(pVCpu);
6587	HM_DISABLE_PREEMPT();
6588
6589	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6590	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6591	{
6592	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS));
6593	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6594	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6595	}
6596
6597	HM_RESTORE_PREEMPT();
6598	VMMRZCallRing3Enable(pVCpu);
6599
6600	return VINF_SUCCESS;
6601	}
6602
6603
6604	/**
6605	* Saves the auto load/store'd guest MSRs from the current VMCS into
6606	* the guest-CPU context.
6607	*
6608	* @returns VBox status code.
6609	* @param pVCpu The cross context virtual CPU structure.
6610	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6611	* out-of-sync. Make sure to update the required fields
6612	* before using them.
6613	*
6614	* @remarks No-long-jump zone!!!
6615	*/
6616	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6617	{
6618	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6619	return VINF_SUCCESS;
6620
6621	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6622	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6623	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6624	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6625	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6626	{
6627	switch (pMsr->u32Msr)
6628	{
6629	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6630	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6631	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6632	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6633	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6634	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6635	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6636	break;
6637
6638	default:
6639	{
6640	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6641	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6642	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6643	}
6644	}
6645	}
6646
6647	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6648	return VINF_SUCCESS;
6649	}
6650
6651
6652	/**
6653	* Saves the guest control registers from the current VMCS into the guest-CPU
6654	* context.
6655	*
6656	* @returns VBox status code.
6657	* @param pVCpu The cross context virtual CPU structure.
6658	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6659	* out-of-sync. Make sure to update the required fields
6660	* before using them.
6661	*
6662	* @remarks No-long-jump zone!!!
6663	*/
6664	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6665	{
6666	/* Guest CR0. Guest FPU. */
6667	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6668	AssertRCReturn(rc, rc);
6669
6670	/* Guest CR4. */
6671	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6672	AssertRCReturn(rc, rc);
6673
6674	/* Guest CR2 - updated always during the world-switch or in #PF. */
6675	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6676	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6677	{
6678	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6679	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6680	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6681
6682	PVM pVM = pVCpu->CTX_SUFF(pVM);
6683	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6684	\|\| ( pVM->hm.s.fNestedPaging
6685	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6686	{
6687	uint64_t u64Val = 0;
6688	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6689	if (pMixedCtx->cr3 != u64Val)
6690	{
6691	CPUMSetGuestCR3(pVCpu, u64Val);
6692	if (VMMRZCallRing3IsEnabled(pVCpu))
6693	{
6694	PGMUpdateCR3(pVCpu, u64Val);
6695	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6696	}
6697	else
6698	{
6699	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6700	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6701	}
6702	}
6703
6704	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6705	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6706	{
6707	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6708	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6709	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6710	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6711	AssertRCReturn(rc, rc);
6712
6713	if (VMMRZCallRing3IsEnabled(pVCpu))
6714	{
6715	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6716	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6717	}
6718	else
6719	{
6720	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6721	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6722	}
6723	}
6724	}
6725
6726	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6727	}
6728
6729	/*
6730	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6731	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6732	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6733	*
6734	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6735	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6736	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6737	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6738	*
6739	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6740	*/
6741	if (VMMRZCallRing3IsEnabled(pVCpu))
6742	{
6743	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6744	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6745
6746	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6747	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6748
6749	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6750	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6751	}
6752
6753	return rc;
6754	}
6755
6756
6757	/**
6758	* Saves a guest segment register from the current VMCS into the guest-CPU
6759	* context.
6760	*
6761	* @returns VBox status code.
6762	* @param pVCpu The cross context virtual CPU structure.
6763	* @param idxSel Index of the selector in the VMCS.
6764	* @param idxLimit Index of the segment limit in the VMCS.
6765	* @param idxBase Index of the segment base in the VMCS.
6766	* @param idxAccess Index of the access rights of the segment in the VMCS.
6767	* @param pSelReg Pointer to the segment selector.
6768	*
6769	* @remarks No-long-jump zone!!!
6770	* @remarks Never call this function directly!!! Use the
6771	* HMVMX_SAVE_SREG() macro as that takes care of whether to read
6772	* from the VMCS cache or not.
6773	*/
6774	static int hmR0VmxSaveSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6775	PCPUMSELREG pSelReg)
6776	{
6777	NOREF(pVCpu);
6778
6779	uint32_t u32Val = 0;
6780	int rc = VMXReadVmcs32(idxSel, &u32Val);
6781	AssertRCReturn(rc, rc);
6782	pSelReg->Sel = (uint16_t)u32Val;
6783	pSelReg->ValidSel = (uint16_t)u32Val;
6784	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6785
6786	rc = VMXReadVmcs32(idxLimit, &u32Val);
6787	AssertRCReturn(rc, rc);
6788	pSelReg->u32Limit = u32Val;
6789
6790	uint64_t u64Val = 0;
6791	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6792	AssertRCReturn(rc, rc);
6793	pSelReg->u64Base = u64Val;
6794
6795	rc = VMXReadVmcs32(idxAccess, &u32Val);
6796	AssertRCReturn(rc, rc);
6797	pSelReg->Attr.u = u32Val;
6798
6799	/*
6800	* If VT-x marks the segment as unusable, most other bits remain undefined:
6801	* - For CS the L, D and G bits have meaning.
6802	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6803	* - For the remaining data segments no bits are defined.
6804	*
6805	* The present bit and the unusable bit has been observed to be set at the
6806	* same time (the selector was supposed to be invalid as we started executing
6807	* a V8086 interrupt in ring-0).
6808	*
6809	* What should be important for the rest of the VBox code, is that the P bit is
6810	* cleared. Some of the other VBox code recognizes the unusable bit, but
6811	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6812	* safe side here, we'll strip off P and other bits we don't care about. If
6813	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6814	*
6815	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6816	*/
6817	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6818	{
6819	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6820
6821	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6822	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6823	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6824
6825	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6826	#ifdef DEBUG_bird
6827	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6828	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6829	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6830	#endif
6831	}
6832	return VINF_SUCCESS;
6833	}
6834
6835	/**
6836	* Saves the guest segment registers from the current VMCS into the guest-CPU
6837	* context.
6838	*
6839	* @returns VBox status code.
6840	* @param pVCpu The cross context virtual CPU structure.
6841	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6842	* out-of-sync. Make sure to update the required fields
6843	* before using them.
6844	*
6845	* @remarks No-long-jump zone!!!
6846	*/
6847	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6848	{
6849	/* Guest segment registers. */
6850	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6851	{
6852	/** @todo r=ramshankar: Why do we save CR0 here? */
6853	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6854	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6855	AssertRCReturn(rc, rc);
6856
6857	rc = HMVMX_SAVE_SREG(CS, &pMixedCtx->cs);
6858	rc \|= HMVMX_SAVE_SREG(SS, &pMixedCtx->ss);
6859	rc \|= HMVMX_SAVE_SREG(DS, &pMixedCtx->ds);
6860	rc \|= HMVMX_SAVE_SREG(ES, &pMixedCtx->es);
6861	rc \|= HMVMX_SAVE_SREG(FS, &pMixedCtx->fs);
6862	rc \|= HMVMX_SAVE_SREG(GS, &pMixedCtx->gs);
6863	AssertRCReturn(rc, rc);
6864
6865	/* Restore segment attributes for real-on-v86 mode hack. */
6866	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6867	{
6868	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6869	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6870	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6871	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6872	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6873	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6874	}
6875	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6876	}
6877
6878	return VINF_SUCCESS;
6879	}
6880
6881
6882	/**
6883	* Saves the guest SS register from the current VMCS into the guest-CPU context.
6884	*
6885	* @returns VBox status code.
6886	* @param pVCpu The cross context virtual CPU structure.
6887	* @remarks No-long-jump zone!!!
6888	*/
6889	static int hmR0VmxSaveGuestCs(PVMCPU pVCpu)
6890	{
6891	/** @todo optimize this? */
6892	return hmR0VmxSaveGuestSegmentRegs(pVCpu, &pVCpu->cpum.GstCtx);
6893	}
6894
6895
6896	/**
6897	* Saves the guest descriptor table registers and task register from the current
6898	* VMCS into the guest-CPU context.
6899	*
6900	* @returns VBox status code.
6901	* @param pVCpu The cross context virtual CPU structure.
6902	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6903	* out-of-sync. Make sure to update the required fields
6904	* before using them.
6905	*
6906	* @remarks No-long-jump zone!!!
6907	*/
6908	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6909	{
6910	int rc = VINF_SUCCESS;
6911
6912	/* Guest LDTR. */
6913	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6914	{
6915	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6916	rc = HMVMX_SAVE_SREG(LDTR, &pMixedCtx->ldtr);
6917	AssertRCReturn(rc, rc);
6918	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6919	}
6920
6921	/* Guest GDTR. */
6922	uint64_t u64Val = 0;
6923	uint32_t u32Val = 0;
6924	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6925	{
6926	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6927	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6928	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6929	pMixedCtx->gdtr.pGdt = u64Val;
6930	pMixedCtx->gdtr.cbGdt = u32Val;
6931	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6932	}
6933
6934	/* Guest IDTR. */
6935	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6936	{
6937	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6938	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6939	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6940	pMixedCtx->idtr.pIdt = u64Val;
6941	pMixedCtx->idtr.cbIdt = u32Val;
6942	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6943	}
6944
6945	/* Guest TR. */
6946	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6947	{
6948	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6949	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6950	AssertRCReturn(rc, rc);
6951
6952	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6953	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6954	{
6955	rc = HMVMX_SAVE_SREG(TR, &pMixedCtx->tr);
6956	AssertRCReturn(rc, rc);
6957	}
6958	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6959	}
6960	return rc;
6961	}
6962
6963
6964	/**
6965	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6966	* context.
6967	*
6968	* @returns VBox status code.
6969	* @param pVCpu The cross context virtual CPU structure.
6970	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6971	* out-of-sync. Make sure to update the required fields
6972	* before using them.
6973	*
6974	* @remarks No-long-jump zone!!!
6975	*/
6976	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6977	{
6978	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6979	{
6980	if (!pVCpu->hm.s.fUsingHyperDR7)
6981	{
6982	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6983	uint32_t u32Val;
6984	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6985	pMixedCtx->dr[7] = u32Val;
6986	}
6987
6988	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6989	}
6990	return VINF_SUCCESS;
6991	}
6992
6993
6994	/**
6995	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6996	*
6997	* @returns VBox status code.
6998	* @param pVCpu The cross context virtual CPU structure.
6999	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7000	* out-of-sync. Make sure to update the required fields
7001	* before using them.
7002	*
7003	* @remarks No-long-jump zone!!!
7004	*/
7005	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7006	{
7007	NOREF(pMixedCtx);
7008
7009	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
7010	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7011	return VINF_SUCCESS;
7012	}
7013
7014
7015	/**
7016	* Worker for VMXR0ImportStateOnDemand.
7017	*
7018	* @returns VBox status code.
7019	* @param pVCpu The cross context virtual CPU structure.
7020	* @param pCtx Pointer to the guest-CPU context.
7021	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7022	*/
7023	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCPUMCTX pCtx, uint64_t fWhat)
7024	{
7025	int rc = VINF_SUCCESS;
7026	PVM pVM = pVCpu->CTX_SUFF(pVM);
7027	uint64_t u64Val;
7028	uint32_t u32Val;
7029	uint32_t u32Shadow;
7030
7031	/*
7032	* Though we can longjmp to ring-3 due to log-flushes here and get re-invoked
7033	* on the ring-3 callback path, there is no real need to.
7034	*/
7035	if (VMMRZCallRing3IsEnabled(pVCpu))
7036	VMMR0LogFlushDisable(pVCpu);
7037	else
7038	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7039	Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
7040
7041	/*
7042	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7043	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7044	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7045	*
7046	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7047	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7048	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7049	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
7050	*
7051	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7052	*/
7053	if (VMMRZCallRing3IsEnabled(pVCpu))
7054	{
7055	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7056	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7057
7058	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7059	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7060
7061	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7062	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7063
7064	VMMR0LogFlushEnable(pVCpu);
7065	}
7066
7067	Assert(!(fWhat & CPUMCTX_EXTRN_KEEPER_HM));
7068	fWhat &= pCtx->fExtrn;
7069
7070	/* If there is nothing more to import, bail early. */
7071	if (!(fWhat & HMVMX_CPUMCTX_EXTRN_ALL))
7072	return VINF_SUCCESS;
7073
7074	/* RIP required while saving interruptibility-state below, see EMSetInhibitInterruptsPC(). */
7075	if (fWhat & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_HM_VMX_INT_STATE))
7076	{
7077	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7078	AssertRCReturn(rc, rc);
7079	pCtx->rip = u64Val;
7080	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_RIP);
7081	}
7082
7083	/* RFLAGS and interruptibility-state required while re-evaluating interrupt injection, see hmR0VmxGetGuestIntrState(). */
7084	if (fWhat & (CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_HM_VMX_INT_STATE))
7085	{
7086	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7087	AssertRCReturn(rc, rc);
7088	pCtx->eflags.u32 = u64Val;
7089	/* Restore eflags for real-on-v86-mode hack. */
7090	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7091	{
7092	Assert(pVM->hm.s.vmx.pRealModeTSS);
7093	pCtx->eflags.Bits.u1VM = 0;
7094	pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
7095	}
7096	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_RFLAGS);
7097	}
7098
7099	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7100	{
7101	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
7102	AssertRCReturn(rc, rc);
7103	if (!u32Val)
7104	{
7105	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7106	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7107
7108	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
7109	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7110	}
7111	else
7112	{
7113	if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
7114	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
7115	{
7116	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
7117	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
7118	}
7119	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7120	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7121
7122	if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
7123	{
7124	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
7125	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7126	}
7127	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
7128	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7129	}
7130	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_HM_VMX_INT_STATE);
7131	}
7132
7133	if (fWhat & CPUMCTX_EXTRN_RSP)
7134	{
7135	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7136	AssertRCReturn(rc, rc);
7137	pCtx->rsp = u64Val;
7138	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_RSP);
7139	}
7140
7141	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7142	{
7143	if (fWhat & CPUMCTX_EXTRN_CS)
7144	{
7145	rc = HMVMX_SAVE_SREG(CS, &pCtx->cs);
7146	AssertRCReturn(rc, rc);
7147	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7148	pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
7149	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_CS);
7150	}
7151	if (fWhat & CPUMCTX_EXTRN_SS)
7152	{
7153	rc = HMVMX_SAVE_SREG(SS, &pCtx->ss);
7154	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7155	pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
7156	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_SS);
7157	}
7158	if (fWhat & CPUMCTX_EXTRN_DS)
7159	{
7160	rc = HMVMX_SAVE_SREG(DS, &pCtx->ds);
7161	AssertRCReturn(rc, rc);
7162	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7163	pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
7164	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_DS);
7165	}
7166	if (fWhat & CPUMCTX_EXTRN_ES)
7167	{
7168	rc = HMVMX_SAVE_SREG(ES, &pCtx->es);
7169	AssertRCReturn(rc, rc);
7170	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7171	pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
7172	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_ES);
7173	}
7174	if (fWhat & CPUMCTX_EXTRN_FS)
7175	{
7176	rc = HMVMX_SAVE_SREG(FS, &pCtx->fs);
7177	AssertRCReturn(rc, rc);
7178	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7179	pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
7180	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_FS);
7181	}
7182	if (fWhat & CPUMCTX_EXTRN_GS)
7183	{
7184	rc = HMVMX_SAVE_SREG(GS, &pCtx->gs);
7185	AssertRCReturn(rc, rc);
7186	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7187	pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
7188	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_GS);
7189	}
7190	}
7191
7192	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7193	{
7194	if (fWhat & CPUMCTX_EXTRN_LDTR)
7195	{
7196	rc = HMVMX_SAVE_SREG(LDTR, &pCtx->ldtr);
7197	AssertRCReturn(rc, rc);
7198	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_LDTR);
7199	}
7200
7201	if (fWhat & CPUMCTX_EXTRN_GDTR)
7202	{
7203	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7204	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7205	AssertRCReturn(rc, rc);
7206	pCtx->gdtr.pGdt = u64Val;
7207	pCtx->gdtr.cbGdt = u32Val;
7208	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_GDTR);
7209	}
7210
7211	/* Guest IDTR. */
7212	if (fWhat & CPUMCTX_EXTRN_IDTR)
7213	{
7214	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7215	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7216	AssertRCReturn(rc, rc);
7217	pCtx->idtr.pIdt = u64Val;
7218	pCtx->idtr.cbIdt = u32Val;
7219	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_IDTR);
7220	}
7221
7222	/* Guest TR. */
7223	if (fWhat & CPUMCTX_EXTRN_TR)
7224	{
7225	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
7226	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7227	{
7228	rc = HMVMX_SAVE_SREG(TR, &pCtx->tr);
7229	AssertRCReturn(rc, rc);
7230	}
7231	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_TR);
7232	}
7233	}
7234
7235	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7236	{
7237	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7238	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7239	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7240	pCtx->SysEnter.cs = u32Val;
7241	AssertRCReturn(rc, rc);
7242	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_SYSENTER_MSRS);
7243	}
7244
7245	#if HC_ARCH_BITS == 64
7246	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7247	{
7248	if ( pVM->hm.s.fAllow64BitGuests
7249	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7250	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7251	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_KERNEL_GS_BASE);
7252	}
7253
7254	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7255	{
7256	if ( pVM->hm.s.fAllow64BitGuests
7257	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7258	{
7259	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7260	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7261	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7262	}
7263	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_SYSCALL_MSRS);
7264	}
7265	#endif
7266
7267	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7268	#if HC_ARCH_BITS == 32
7269	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7270	#endif
7271	)
7272	{
7273	PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
7274	uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
7275	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7276	{
7277	switch (pMsr->u32Msr)
7278	{
7279	#if HC_ARCH_BITS == 32
7280	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7281	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7282	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7283	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7284	#endif
7285	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7286	case MSR_K8_TSC_AUX:
7287	{
7288	/* CPUMSetGuestTscAux alters fExtrn without using atomics, so disable preemption temporarily. */
7289	HM_DISABLE_PREEMPT();
7290	CPUMSetGuestTscAux(pVCpu, pMsr->u64Value);
7291	HM_RESTORE_PREEMPT();
7292	break;
7293	}
7294	default:
7295	{
7296	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
7297	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7298	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7299	}
7300	}
7301	}
7302	ASMAtomicUoAndU64(&pCtx->fExtrn, ~( CPUMCTX_EXTRN_TSC_AUX
7303	\| CPUMCTX_EXTRN_OTHER_MSRS
7304	#if HC_ARCH_BITS == 32
7305	\| CPUMCTX_EXTRN_KERNEL_GS_BASE
7306	\| CPUMCTX_EXTRN_SYSCALL_MSRS
7307	#endif
7308	));
7309	}
7310
7311	if (fWhat & CPUMCTX_EXTRN_DR7)
7312	{
7313	if (!pVCpu->hm.s.fUsingHyperDR7)
7314	{
7315	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7316	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7317	AssertRCReturn(rc, rc);
7318	pCtx->dr[7] = u32Val;
7319	}
7320	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_DR7);
7321	}
7322
7323	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7324	{
7325	/* CR0 required for saving CR3 below, see CPUMIsGuestPagingEnabledEx(). */
7326	if (fWhat & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3))
7327	{
7328	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7329	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7330	AssertRCReturn(rc, rc);
7331	u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR0Mask)
7332	\| (u32Shadow & pVCpu->hm.s.vmx.u32CR0Mask);
7333	CPUMSetGuestCR0(pVCpu, u32Val);
7334	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_CR0);
7335	}
7336
7337	/* CR4 required for saving CR3 below, see CPUMIsGuestInPAEModeEx(). */
7338	if (fWhat & (CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3))
7339	{
7340	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7341	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7342	AssertRCReturn(rc, rc);
7343	u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR4Mask)
7344	\| (u32Shadow & pVCpu->hm.s.vmx.u32CR4Mask);
7345	CPUMSetGuestCR4(pVCpu, u32Val);
7346	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_CR4);
7347	}
7348
7349	if (fWhat & CPUMCTX_EXTRN_CR3)
7350	{
7351	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7352	\|\| ( pVM->hm.s.fNestedPaging
7353	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7354	{
7355	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7356	if (pCtx->cr3 != u64Val)
7357	{
7358	CPUMSetGuestCR3(pVCpu, u64Val);
7359	if (VMMRZCallRing3IsEnabled(pVCpu))
7360	{
7361	PGMUpdateCR3(pVCpu, u64Val);
7362	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7363	}
7364	else
7365	{
7366	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
7367	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7368	}
7369	}
7370
7371	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
7372	if (CPUMIsGuestInPAEModeEx(pCtx))
7373	{
7374	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7375	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7376	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7377	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7378	AssertRCReturn(rc, rc);
7379
7380	if (VMMRZCallRing3IsEnabled(pVCpu))
7381	{
7382	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7383	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7384	}
7385	else
7386	{
7387	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
7388	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7389	}
7390	}
7391	}
7392	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_CR3);
7393	}
7394	}
7395
7396	/* If everything has been imported, clear the HM keeper bit. */
7397	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7398	{
7399	ASMAtomicUoAndU64(&pCtx->fExtrn, ~CPUMCTX_EXTRN_KEEPER_HM);
7400	Assert(!pCtx->fExtrn);
7401	}
7402
7403	return VINF_SUCCESS;
7404	}
7405
7406
7407	/**
7408	* Saves the guest state from the VMCS into the guest-CPU context.
7409	*
7410	* @returns VBox status code.
7411	* @param pVCpu The cross context virtual CPU structure.
7412	* @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
7413	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7414	*/
7415	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, PCPUMCTX pCtx, uint64_t fWhat)
7416	{
7417	return hmR0VmxImportGuestState(pVCpu, pCtx, fWhat);
7418	}
7419
7420
7421	/**
7422	* Saves the entire guest state from the currently active VMCS into the
7423	* guest-CPU context.
7424	*
7425	* This essentially VMREADs all guest-data.
7426	*
7427	* @returns VBox status code.
7428	* @param pVCpu The cross context virtual CPU structure.
7429	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7430	* out-of-sync. Make sure to update the required fields
7431	* before using them.
7432	*/
7433	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7434	{
7435	Assert(pVCpu);
7436	Assert(pMixedCtx);
7437
7438	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7439	return VINF_SUCCESS;
7440
7441	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7442	again on the ring-3 callback path, there is no real need to. */
7443	if (VMMRZCallRing3IsEnabled(pVCpu))
7444	VMMR0LogFlushDisable(pVCpu);
7445	else
7446	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7447	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7448
7449	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7450	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7451
7452	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7453	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7454
7455	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7456	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7457
7458	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7459	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7460
7461	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7462	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7463
7464	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7465	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7466
7467	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7468	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7469
7470	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7471	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7472
7473	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7474	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7475
7476	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7477	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7478
7479	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7480	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7481	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7482
7483	if (VMMRZCallRing3IsEnabled(pVCpu))
7484	VMMR0LogFlushEnable(pVCpu);
7485
7486	return VINF_SUCCESS;
7487	}
7488
7489
7490	/**
7491	* Saves basic guest registers needed for IEM instruction execution.
7492	*
7493	* @returns VBox status code (OR-able).
7494	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7495	* @param pMixedCtx Pointer to the CPU context of the guest.
7496	* @param fMemory Whether the instruction being executed operates on
7497	* memory or not. Only CR0 is synced up if clear.
7498	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7499	*/
7500	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7501	{
7502	/*
7503	* We assume all general purpose registers other than RSP are available.
7504	*
7505	* - RIP is a must, as it will be incremented or otherwise changed.
7506	* - RFLAGS are always required to figure the CPL.
7507	* - RSP isn't always required, however it's a GPR, so frequently required.
7508	* - SS and CS are the only segment register needed if IEM doesn't do memory
7509	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7510	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7511	* be required for memory accesses.
7512	*
7513	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7514	*/
7515	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7516	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7517	if (fNeedRsp)
7518	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7519	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /** @todo Only CS and SS are required here. */
7520	if (!fMemory)
7521	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7522	else
7523	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7524	AssertRCReturn(rc, rc);
7525	return rc;
7526	}
7527
7528
7529	/**
7530	* Saves guest registers needed for IEM instruction interpretation.
7531	*
7532	* @returns VBox status code (OR-able).
7533	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7534	*/
7535	static int hmR0VmxSaveGuestRegsForIemInterpreting(PVMCPU pVCpu)
7536	{
7537	/*
7538	* Our goal here is IEM_CPUMCTX_EXTRN_MUST_MASK.
7539	*
7540	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7541	*/
7542	#if 0 /* later with CPUMCTX_EXTRN_XXX */
7543	int rc = hmR0VmxSaveGuestRip(pVCpu, &pVCpu->cpum.GstCtx);
7544	rc \|= hmR0VmxSaveGuestRflags(pVCpu, &pVCpu->cpum.GstCtx);
7545	rc \|= hmR0VmxSaveGuestRsp(pVCpu, &pVCpu->cpum.GstCtx);
7546	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, &pVCpu->cpum.GstCtx); /** @todo Only CS and SS are strictly required here. */
7547	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, &pVCpu->cpum.GstCtx); /** @todo We don't need CR2 here. */
7548	rc \|= hmR0VmxSaveGuestApicState(pVCpu, &pVCpu->cpum.GstCtx); /** @todo Only TPR is needed here. */
7549	rc \|= hmR0VmxSaveGuestDR7(pVCpu, &pVCpu->cpum.GstCtx);
7550	/* EFER is always up to date. */
7551	AssertRCReturn(rc, rc);
7552	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST - fixme); /** @todo fix me */
7553	#else
7554	int rc = hmR0VmxSaveGuestState(pVCpu, &pVCpu->cpum.GstCtx);
7555	AssertRCReturn(rc, rc);
7556	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7557	#endif
7558
7559	return rc;
7560	}
7561
7562
7563	/**
7564	* Ensures that we've got a complete basic guest-context.
7565	*
7566	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7567	* is for the interpreter.
7568	*
7569	* @returns VBox status code.
7570	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7571	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7572	* needing to be synced in.
7573	* @thread EMT(pVCpu)
7574	*/
7575	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7576	{
7577	/* Note! Since this is only applicable to VT-x, the implementation is placed
7578	in the VT-x part of the sources instead of the generic stuff. */
7579	int rc;
7580	PVM pVM = pVCpu->CTX_SUFF(pVM);
7581	if ( pVM->hm.s.vmx.fSupported
7582	&& VM_IS_HM_ENABLED(pVM))
7583	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7584	else
7585	rc = VINF_SUCCESS;
7586
7587	/*
7588	* For now, imply that the caller might change everything too. Do this after
7589	* saving the guest state so as to not trigger assertions.
7590	*
7591	* This is required for AMD-V too as it too only selectively re-loads changed
7592	* guest state back in to the VMCB.
7593	*/
7594	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7595	return rc;
7596	}
7597
7598
7599	/**
7600	* Check per-VM and per-VCPU force flag actions that require us to go back to
7601	* ring-3 for one reason or another.
7602	*
7603	* @returns Strict VBox status code (i.e. informational status codes too)
7604	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7605	* ring-3.
7606	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7607	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7608	* interrupts)
7609	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7610	* all EMTs to be in ring-3.
7611	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7612	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7613	* to the EM loop.
7614	*
7615	* @param pVM The cross context VM structure.
7616	* @param pVCpu The cross context virtual CPU structure.
7617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7618	* out-of-sync. Make sure to update the required fields
7619	* before using them.
7620	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7621	*/
7622	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7623	{
7624	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7625
7626	/*
7627	* Anything pending? Should be more likely than not if we're doing a good job.
7628	*/
7629	if ( !fStepping
7630	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7631	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7632	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7633	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7634	return VINF_SUCCESS;
7635
7636	/* We need the control registers now, make sure the guest-CPU context is updated. */
7637	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7638	AssertRCReturn(rc3, rc3);
7639
7640	/** @todo r=ramshankar: VMCPU_FF_HM_UPDATE_CR3 and VMCPU_FF_HM_UPDATE_PAE_PDPES
7641	* are not part of VMCPU_FF_HP_R0_PRE_HM_MASK. Hence, the two if
7642	* statements below won't ever be entered. Consider removing it or
7643	* determine if it is necessary to add these flags to VMCPU_FF_HP_R0_PRE_HM_MASK. */
7644	/* Pending HM CR3 sync. */
7645	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7646	{
7647	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7648	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7649	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7650	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7651	}
7652
7653	/* Pending HM PAE PDPEs. */
7654	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7655	{
7656	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7657	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7658	}
7659
7660	/* Pending PGM C3 sync. */
7661	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7662	{
7663	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7664	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7665	if (rcStrict2 != VINF_SUCCESS)
7666	{
7667	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7668	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7669	return rcStrict2;
7670	}
7671	}
7672
7673	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7674	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7675	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7676	{
7677	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7678	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7679	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7680	return rc2;
7681	}
7682
7683	/* Pending VM request packets, such as hardware interrupts. */
7684	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7685	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7686	{
7687	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7688	return VINF_EM_PENDING_REQUEST;
7689	}
7690
7691	/* Pending PGM pool flushes. */
7692	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7693	{
7694	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7695	return VINF_PGM_POOL_FLUSH_PENDING;
7696	}
7697
7698	/* Pending DMA requests. */
7699	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7700	{
7701	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7702	return VINF_EM_RAW_TO_R3;
7703	}
7704
7705	return VINF_SUCCESS;
7706	}
7707
7708
7709	/**
7710	* Converts any TRPM trap into a pending HM event. This is typically used when
7711	* entering from ring-3 (not longjmp returns).
7712	*
7713	* @param pVCpu The cross context virtual CPU structure.
7714	*/
7715	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7716	{
7717	Assert(TRPMHasTrap(pVCpu));
7718	Assert(!pVCpu->hm.s.Event.fPending);
7719
7720	uint8_t uVector;
7721	TRPMEVENT enmTrpmEvent;
7722	RTGCUINT uErrCode;
7723	RTGCUINTPTR GCPtrFaultAddress;
7724	uint8_t cbInstr;
7725
7726	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7727	AssertRC(rc);
7728
7729	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7730	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7731	if (enmTrpmEvent == TRPM_TRAP)
7732	{
7733	switch (uVector)
7734	{
7735	case X86_XCPT_NMI:
7736	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7737	break;
7738
7739	case X86_XCPT_BP:
7740	case X86_XCPT_OF:
7741	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7742	break;
7743
7744	case X86_XCPT_PF:
7745	case X86_XCPT_DF:
7746	case X86_XCPT_TS:
7747	case X86_XCPT_NP:
7748	case X86_XCPT_SS:
7749	case X86_XCPT_GP:
7750	case X86_XCPT_AC:
7751	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7752	RT_FALL_THRU();
7753	default:
7754	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7755	break;
7756	}
7757	}
7758	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7759	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7760	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7761	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7762	else
7763	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7764
7765	rc = TRPMResetTrap(pVCpu);
7766	AssertRC(rc);
7767	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7768	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7769
7770	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7771	}
7772
7773
7774	/**
7775	* Converts the pending HM event into a TRPM trap.
7776	*
7777	* @param pVCpu The cross context virtual CPU structure.
7778	*/
7779	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7780	{
7781	Assert(pVCpu->hm.s.Event.fPending);
7782
7783	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7784	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7785	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7786	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7787
7788	/* If a trap was already pending, we did something wrong! */
7789	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7790
7791	TRPMEVENT enmTrapType;
7792	switch (uVectorType)
7793	{
7794	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7795	enmTrapType = TRPM_HARDWARE_INT;
7796	break;
7797
7798	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7799	enmTrapType = TRPM_SOFTWARE_INT;
7800	break;
7801
7802	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7803	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7804	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7805	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7806	enmTrapType = TRPM_TRAP;
7807	break;
7808
7809	default:
7810	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7811	enmTrapType = TRPM_32BIT_HACK;
7812	break;
7813	}
7814
7815	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7816
7817	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7818	AssertRC(rc);
7819
7820	if (fErrorCodeValid)
7821	TRPMSetErrorCode(pVCpu, uErrorCode);
7822
7823	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7824	&& uVector == X86_XCPT_PF)
7825	{
7826	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7827	}
7828	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7829	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7830	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7831	{
7832	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7833	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7834	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7835	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7836	}
7837
7838	/* Clear any pending events from the VMCS. */
7839	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7840	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7841
7842	/* We're now done converting the pending event. */
7843	pVCpu->hm.s.Event.fPending = false;
7844	}
7845
7846
7847	/**
7848	* Does the necessary state syncing before returning to ring-3 for any reason
7849	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7850	*
7851	* @returns VBox status code.
7852	* @param pVCpu The cross context virtual CPU structure.
7853	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7854	* be out-of-sync. Make sure to update the required
7855	* fields before using them.
7856	* @param fSaveGuestState Whether to save the guest state or not.
7857	*
7858	* @remarks No-long-jmp zone!!!
7859	*/
7860	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7861	{
7862	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7863	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7864
7865	RTCPUID idCpu = RTMpCpuId();
7866	Log4Func(("HostCpuId=%u\n", idCpu));
7867
7868	/*
7869	* !!! IMPORTANT !!!
7870	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7871	*/
7872
7873	/* Save the guest state if necessary. */
7874	if ( fSaveGuestState
7875	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7876	{
7877	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7878	AssertRCReturn(rc, rc);
7879	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7880	}
7881
7882	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7883	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7884	{
7885	/* We shouldn't reload CR0 without saving it first. */
7886	if (!fSaveGuestState)
7887	{
7888	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7889	AssertRCReturn(rc, rc);
7890	}
7891	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7892	}
7893
7894	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7895	#ifdef VBOX_STRICT
7896	if (CPUMIsHyperDebugStateActive(pVCpu))
7897	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7898	#endif
7899	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7900	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7901	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7902	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7903
7904	#if HC_ARCH_BITS == 64
7905	/* Restore host-state bits that VT-x only restores partially. */
7906	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7907	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7908	{
7909	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7910	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7911	}
7912	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7913	#endif
7914
7915	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7916	if (pVCpu->hm.s.vmx.fLazyMsrs)
7917	{
7918	/* We shouldn't reload the guest MSRs without saving it first. */
7919	if (!fSaveGuestState)
7920	{
7921	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7922	AssertRCReturn(rc, rc);
7923	}
7924	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7925	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7926	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7927	}
7928
7929	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7930	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7931
7932	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7933	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7934	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7935	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7936	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7937	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7938	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7939	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7940
7941	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7942
7943	/** @todo This partially defeats the purpose of having preemption hooks.
7944	* The problem is, deregistering the hooks should be moved to a place that
7945	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7946	* context.
7947	*/
7948	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7949	{
7950	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7951	AssertRCReturn(rc, rc);
7952
7953	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7954	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7955	}
7956	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7957	NOREF(idCpu);
7958
7959	return VINF_SUCCESS;
7960	}
7961
7962
7963	/**
7964	* Leaves the VT-x session.
7965	*
7966	* @returns VBox status code.
7967	* @param pVCpu The cross context virtual CPU structure.
7968	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7969	* out-of-sync. Make sure to update the required fields
7970	* before using them.
7971	*
7972	* @remarks No-long-jmp zone!!!
7973	*/
7974	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7975	{
7976	HM_DISABLE_PREEMPT();
7977	HMVMX_ASSERT_CPU_SAFE();
7978	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7979	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7980
7981	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7982	and done this from the VMXR0ThreadCtxCallback(). */
7983	if (!pVCpu->hm.s.fLeaveDone)
7984	{
7985	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7986	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7987	pVCpu->hm.s.fLeaveDone = true;
7988	}
7989	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7990
7991	/*
7992	* !!! IMPORTANT !!!
7993	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7994	*/
7995
7996	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7997	/** @todo Deregistering here means we need to VMCLEAR always
7998	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7999	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8000	VMMR0ThreadCtxHookDisable(pVCpu);
8001
8002	/* Leave HM context. This takes care of local init (term). */
8003	int rc = HMR0LeaveCpu(pVCpu);
8004
8005	HM_RESTORE_PREEMPT();
8006	return rc;
8007	}
8008
8009
8010	/**
8011	* Does the necessary state syncing before doing a longjmp to ring-3.
8012	*
8013	* @returns VBox status code.
8014	* @param pVCpu The cross context virtual CPU structure.
8015	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8016	* out-of-sync. Make sure to update the required fields
8017	* before using them.
8018	*
8019	* @remarks No-long-jmp zone!!!
8020	*/
8021	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8022	{
8023	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
8024	}
8025
8026
8027	/**
8028	* Take necessary actions before going back to ring-3.
8029	*
8030	* An action requires us to go back to ring-3. This function does the necessary
8031	* steps before we can safely return to ring-3. This is not the same as longjmps
8032	* to ring-3, this is voluntary and prepares the guest so it may continue
8033	* executing outside HM (recompiler/IEM).
8034	*
8035	* @returns VBox status code.
8036	* @param pVM The cross context VM structure.
8037	* @param pVCpu The cross context virtual CPU structure.
8038	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8039	* out-of-sync. Make sure to update the required fields
8040	* before using them.
8041	* @param rcExit The reason for exiting to ring-3. Can be
8042	* VINF_VMM_UNKNOWN_RING3_CALL.
8043	*/
8044	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
8045	{
8046	Assert(pVM);
8047	Assert(pVCpu);
8048	Assert(pMixedCtx);
8049	HMVMX_ASSERT_PREEMPT_SAFE();
8050
8051	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8052	{
8053	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
8054	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
8055	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8056	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8057	}
8058
8059	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8060	VMMRZCallRing3Disable(pVCpu);
8061	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
8062
8063	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
8064	if (pVCpu->hm.s.Event.fPending)
8065	{
8066	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8067	Assert(!pVCpu->hm.s.Event.fPending);
8068	}
8069
8070	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
8071	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
8072
8073	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8074	and if we're injecting an event we should have a TRPM trap pending. */
8075	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8076	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
8077	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8078	#endif
8079
8080	/* Save guest state and restore host state bits. */
8081	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
8082	AssertRCReturn(rc, rc);
8083	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8084	/* Thread-context hooks are unregistered at this point!!! */
8085
8086	/* Sync recompiler state. */
8087	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8088	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8089	\| CPUM_CHANGED_LDTR
8090	\| CPUM_CHANGED_GDTR
8091	\| CPUM_CHANGED_IDTR
8092	\| CPUM_CHANGED_TR
8093	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8094	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8095	if ( pVM->hm.s.fNestedPaging
8096	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
8097	{
8098	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8099	}
8100
8101	Assert(!pVCpu->hm.s.fClearTrapFlag);
8102
8103	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8104	if (rcExit != VINF_EM_RAW_INTERRUPT)
8105	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8106
8107	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8108
8109	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8110	VMMRZCallRing3RemoveNotification(pVCpu);
8111	VMMRZCallRing3Enable(pVCpu);
8112
8113	return rc;
8114	}
8115
8116
8117	/**
8118	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8119	* longjump to ring-3 and possibly get preempted.
8120	*
8121	* @returns VBox status code.
8122	* @param pVCpu The cross context virtual CPU structure.
8123	* @param enmOperation The operation causing the ring-3 longjump.
8124	* @param pvUser Opaque pointer to the guest-CPU context. The data
8125	* may be out-of-sync. Make sure to update the required
8126	* fields before using them.
8127	*/
8128	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8129	{
8130	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8131	{
8132	/*
8133	* !!! IMPORTANT !!!
8134	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8135	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8136	*/
8137	VMMRZCallRing3RemoveNotification(pVCpu);
8138	VMMRZCallRing3Disable(pVCpu);
8139	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8140	RTThreadPreemptDisable(&PreemptState);
8141
8142	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8143	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8144
8145	#if HC_ARCH_BITS == 64
8146	/* Restore host-state bits that VT-x only restores partially. */
8147	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8148	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8149	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8150	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8151	#endif
8152	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8153	if (pVCpu->hm.s.vmx.fLazyMsrs)
8154	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8155
8156	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8157	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
8158	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8159	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
8160	{
8161	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8162	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
8163	}
8164
8165	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8166	VMMR0ThreadCtxHookDisable(pVCpu);
8167	HMR0LeaveCpu(pVCpu);
8168	RTThreadPreemptRestore(&PreemptState);
8169	return VINF_SUCCESS;
8170	}
8171
8172	Assert(pVCpu);
8173	Assert(pvUser);
8174	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8175	HMVMX_ASSERT_PREEMPT_SAFE();
8176
8177	VMMRZCallRing3Disable(pVCpu);
8178	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8179
8180	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
8181	enmOperation));
8182
8183	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
8184	AssertRCReturn(rc, rc);
8185
8186	VMMRZCallRing3Enable(pVCpu);
8187	return VINF_SUCCESS;
8188	}
8189
8190
8191	/**
8192	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8193	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8194	*
8195	* @param pVCpu The cross context virtual CPU structure.
8196	*/
8197	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
8198	{
8199	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
8200	{
8201	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
8202	{
8203	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8204	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8205	AssertRC(rc);
8206	Log4(("Setup interrupt-window exiting\n"));
8207	}
8208	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
8209	}
8210
8211
8212	/**
8213	* Clears the interrupt-window exiting control in the VMCS.
8214	*
8215	* @param pVCpu The cross context virtual CPU structure.
8216	*/
8217	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
8218	{
8219	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8220	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8221	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8222	AssertRC(rc);
8223	Log4(("Cleared interrupt-window exiting\n"));
8224	}
8225
8226
8227	/**
8228	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8229	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8230	*
8231	* @param pVCpu The cross context virtual CPU structure.
8232	*/
8233	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
8234	{
8235	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
8236	{
8237	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
8238	{
8239	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
8240	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8241	AssertRC(rc);
8242	Log4(("Setup NMI-window exiting\n"));
8243	}
8244	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8245	}
8246
8247
8248	/**
8249	* Clears the NMI-window exiting control in the VMCS.
8250	*
8251	* @param pVCpu The cross context virtual CPU structure.
8252	*/
8253	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
8254	{
8255	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
8256	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
8257	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8258	AssertRC(rc);
8259	Log4(("Cleared NMI-window exiting\n"));
8260	}
8261
8262
8263	/**
8264	* Evaluates the event to be delivered to the guest and sets it as the pending
8265	* event.
8266	*
8267	* @returns The VT-x guest-interruptibility state.
8268	* @param pVCpu The cross context virtual CPU structure.
8269	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8270	* out-of-sync. Make sure to update the required fields
8271	* before using them.
8272	*/
8273	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8274	{
8275	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8276	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
8277	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
8278	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
8279	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
8280
8281	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
8282	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8283	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8284	Assert(!TRPMHasTrap(pVCpu));
8285
8286	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8287	APICUpdatePendingInterrupts(pVCpu);
8288
8289	/*
8290	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8291	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8292	*/
8293	/** @todo SMI. SMIs take priority over NMIs. */
8294	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8295	{
8296	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8297	if ( !pVCpu->hm.s.Event.fPending
8298	&& !fBlockNmi
8299	&& !fBlockSti
8300	&& !fBlockMovSS)
8301	{
8302	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
8303	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8304	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8305
8306	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8307	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8308	}
8309	else
8310	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
8311	}
8312	/*
8313	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8314	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
8315	*/
8316	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
8317	&& !pVCpu->hm.s.fSingleInstruction)
8318	{
8319	Assert(!DBGFIsStepping(pVCpu));
8320	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8321	AssertRC(rc);
8322	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
8323	if ( !pVCpu->hm.s.Event.fPending
8324	&& !fBlockInt
8325	&& !fBlockSti
8326	&& !fBlockMovSS)
8327	{
8328	uint8_t u8Interrupt;
8329	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8330	if (RT_SUCCESS(rc))
8331	{
8332	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
8333	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8334	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8335
8336	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
8337	}
8338	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8339	{
8340	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8341	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
8342	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8343
8344	/*
8345	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8346	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8347	* need to re-set this force-flag here.
8348	*/
8349	}
8350	else
8351	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8352	}
8353	else
8354	hmR0VmxSetIntWindowExitVmcs(pVCpu);
8355	}
8356
8357	return uIntrState;
8358	}
8359
8360
8361	/**
8362	* Sets a pending-debug exception to be delivered to the guest if the guest is
8363	* single-stepping in the VMCS.
8364	*
8365	* @param pVCpu The cross context virtual CPU structure.
8366	*/
8367	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
8368	{
8369	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
8370	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
8371	AssertRC(rc);
8372	}
8373
8374
8375	/**
8376	* Injects any pending events into the guest if the guest is in a state to
8377	* receive them.
8378	*
8379	* @returns Strict VBox status code (i.e. informational status codes too).
8380	* @param pVCpu The cross context virtual CPU structure.
8381	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8382	* out-of-sync. Make sure to update the required fields
8383	* before using them.
8384	* @param uIntrState The VT-x guest-interruptibility state.
8385	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
8386	* return VINF_EM_DBG_STEPPED if the event was
8387	* dispatched directly.
8388	*/
8389	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
8390	{
8391	HMVMX_ASSERT_PREEMPT_SAFE();
8392	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8393
8394	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
8395	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
8396
8397	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
8398	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8399	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8400	Assert(!TRPMHasTrap(pVCpu));
8401
8402	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8403	if (pVCpu->hm.s.Event.fPending)
8404	{
8405	/*
8406	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8407	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8408	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8409	*
8410	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8411	*/
8412	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8413	#ifdef VBOX_STRICT
8414	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8415	{
8416	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
8417	Assert(!fBlockInt);
8418	Assert(!fBlockSti);
8419	Assert(!fBlockMovSS);
8420	}
8421	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8422	{
8423	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
8424	Assert(!fBlockSti);
8425	Assert(!fBlockMovSS);
8426	Assert(!fBlockNmi);
8427	}
8428	#endif
8429	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8430	(uint8_t)uIntType));
8431	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
8432	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
8433	fStepping, &uIntrState);
8434	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8435
8436	/* Update the interruptibility-state as it could have been changed by
8437	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
8438	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
8439	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
8440
8441	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8442	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8443	else
8444	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8445	}
8446
8447	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
8448	if ( fBlockSti
8449	\|\| fBlockMovSS)
8450	{
8451	if (!pVCpu->hm.s.fSingleInstruction)
8452	{
8453	/*
8454	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8455	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8456	* See Intel spec. 27.3.4 "Saving Non-Register State".
8457	*/
8458	Assert(!DBGFIsStepping(pVCpu));
8459	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8460	AssertRCReturn(rc2, rc2);
8461	if (pMixedCtx->eflags.Bits.u1TF)
8462	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8463	}
8464	else if (pMixedCtx->eflags.Bits.u1TF)
8465	{
8466	/*
8467	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8468	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8469	*/
8470	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8471	uIntrState = 0;
8472	}
8473	}
8474
8475	/*
8476	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8477	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8478	*/
8479	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8480	AssertRC(rc2);
8481
8482	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8483	NOREF(fBlockMovSS); NOREF(fBlockSti);
8484	return rcStrict;
8485	}
8486
8487
8488	/**
8489	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8490	*
8491	* @param pVCpu The cross context virtual CPU structure.
8492	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8493	* out-of-sync. Make sure to update the required fields
8494	* before using them.
8495	*/
8496	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8497	{
8498	NOREF(pMixedCtx);
8499	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8500	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8501	}
8502
8503
8504	/**
8505	* Injects a double-fault (\#DF) exception into the VM.
8506	*
8507	* @returns Strict VBox status code (i.e. informational status codes too).
8508	* @param pVCpu The cross context virtual CPU structure.
8509	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8510	* out-of-sync. Make sure to update the required fields
8511	* before using them.
8512	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8513	* and should return VINF_EM_DBG_STEPPED if the event
8514	* is injected directly (register modified by us, not
8515	* by hardware on VM-entry).
8516	* @param puIntrState Pointer to the current guest interruptibility-state.
8517	* This interruptibility-state will be updated if
8518	* necessary. This cannot not be NULL.
8519	*/
8520	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8521	{
8522	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8523	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8524	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8525	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8526	fStepping, puIntrState);
8527	}
8528
8529
8530	/**
8531	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8532	*
8533	* @param pVCpu The cross context virtual CPU structure.
8534	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8535	* out-of-sync. Make sure to update the required fields
8536	* before using them.
8537	*/
8538	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8539	{
8540	NOREF(pMixedCtx);
8541	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8542	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8543	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8544	}
8545
8546
8547	/**
8548	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8549	*
8550	* @param pVCpu The cross context virtual CPU structure.
8551	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8552	* out-of-sync. Make sure to update the required fields
8553	* before using them.
8554	* @param cbInstr The value of RIP that is to be pushed on the guest
8555	* stack.
8556	*/
8557	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8558	{
8559	NOREF(pMixedCtx);
8560	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8561	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8562	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8563	}
8564
8565
8566	/**
8567	* Injects a general-protection (\#GP) fault into the VM.
8568	*
8569	* @returns Strict VBox status code (i.e. informational status codes too).
8570	* @param pVCpu The cross context virtual CPU structure.
8571	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8572	* out-of-sync. Make sure to update the required fields
8573	* before using them.
8574	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8575	* mode, i.e. in real-mode it's not valid).
8576	* @param u32ErrorCode The error code associated with the \#GP.
8577	* @param fStepping Whether we're running in
8578	* hmR0VmxRunGuestCodeStep() and should return
8579	* VINF_EM_DBG_STEPPED if the event is injected
8580	* directly (register modified by us, not by
8581	* hardware on VM-entry).
8582	* @param puIntrState Pointer to the current guest interruptibility-state.
8583	* This interruptibility-state will be updated if
8584	* necessary. This cannot not be NULL.
8585	*/
8586	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8587	bool fStepping, uint32_t *puIntrState)
8588	{
8589	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8590	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8591	if (fErrorCodeValid)
8592	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8593	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8594	fStepping, puIntrState);
8595	}
8596
8597
8598	#if 0 /* unused */
8599	/**
8600	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8601	* VM.
8602	*
8603	* @param pVCpu The cross context virtual CPU structure.
8604	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8605	* out-of-sync. Make sure to update the required fields
8606	* before using them.
8607	* @param u32ErrorCode The error code associated with the \#GP.
8608	*/
8609	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8610	{
8611	NOREF(pMixedCtx);
8612	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8613	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8614	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8615	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8616	}
8617	#endif /* unused */
8618
8619
8620	/**
8621	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8622	*
8623	* @param pVCpu The cross context virtual CPU structure.
8624	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8625	* out-of-sync. Make sure to update the required fields
8626	* before using them.
8627	* @param uVector The software interrupt vector number.
8628	* @param cbInstr The value of RIP that is to be pushed on the guest
8629	* stack.
8630	*/
8631	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8632	{
8633	NOREF(pMixedCtx);
8634	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8635	if ( uVector == X86_XCPT_BP
8636	\|\| uVector == X86_XCPT_OF)
8637	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8638	else
8639	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8640	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8641	}
8642
8643
8644	/**
8645	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8646	* stack.
8647	*
8648	* @returns Strict VBox status code (i.e. informational status codes too).
8649	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8650	* @param pVM The cross context VM structure.
8651	* @param pMixedCtx Pointer to the guest-CPU context.
8652	* @param uValue The value to push to the guest stack.
8653	*/
8654	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8655	{
8656	/*
8657	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8658	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8659	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8660	*/
8661	if (pMixedCtx->sp == 1)
8662	return VINF_EM_RESET;
8663	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8664	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8665	AssertRC(rc);
8666	return rc;
8667	}
8668
8669
8670	/**
8671	* Injects an event into the guest upon VM-entry by updating the relevant fields
8672	* in the VM-entry area in the VMCS.
8673	*
8674	* @returns Strict VBox status code (i.e. informational status codes too).
8675	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8676	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8677	*
8678	* @param pVCpu The cross context virtual CPU structure.
8679	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8680	* be out-of-sync. Make sure to update the required
8681	* fields before using them.
8682	* @param u64IntInfo The VM-entry interruption-information field.
8683	* @param cbInstr The VM-entry instruction length in bytes (for
8684	* software interrupts, exceptions and privileged
8685	* software exceptions).
8686	* @param u32ErrCode The VM-entry exception error code.
8687	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8688	* @param puIntrState Pointer to the current guest interruptibility-state.
8689	* This interruptibility-state will be updated if
8690	* necessary. This cannot not be NULL.
8691	* @param fStepping Whether we're running in
8692	* hmR0VmxRunGuestCodeStep() and should return
8693	* VINF_EM_DBG_STEPPED if the event is injected
8694	* directly (register modified by us, not by
8695	* hardware on VM-entry).
8696	*
8697	* @remarks Requires CR0!
8698	*/
8699	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8700	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8701	uint32_t *puIntrState)
8702	{
8703	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8704	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8705	Assert(puIntrState);
8706	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8707
8708	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8709	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8710
8711	#ifdef VBOX_STRICT
8712	/*
8713	* Validate the error-code-valid bit for hardware exceptions.
8714	* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8715	*/
8716	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8717	&& !CPUMIsGuestInRealModeEx(pMixedCtx))
8718	{
8719	switch (uVector)
8720	{
8721	case X86_XCPT_PF:
8722	case X86_XCPT_DF:
8723	case X86_XCPT_TS:
8724	case X86_XCPT_NP:
8725	case X86_XCPT_SS:
8726	case X86_XCPT_GP:
8727	case X86_XCPT_AC:
8728	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8729	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8730	RT_FALL_THRU();
8731	default:
8732	break;
8733	}
8734	}
8735	#endif
8736
8737	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8738	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8739	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8740
8741	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8742
8743	/* We require CR0 to check if the guest is in real-mode. */
8744	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8745	AssertRCReturn(rc, rc);
8746
8747	/*
8748	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8749	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8750	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8751	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8752	*/
8753	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8754	{
8755	PVM pVM = pVCpu->CTX_SUFF(pVM);
8756	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8757	{
8758	Assert(PDMVmmDevHeapIsEnabled(pVM));
8759	Assert(pVM->hm.s.vmx.pRealModeTSS);
8760
8761	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8762	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8763	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8764	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8765	AssertRCReturn(rc, rc);
8766	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8767
8768	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8769	size_t const cbIdtEntry = sizeof(X86IDTR16);
8770	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8771	{
8772	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8773	if (uVector == X86_XCPT_DF)
8774	return VINF_EM_RESET;
8775
8776	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8777	if (uVector == X86_XCPT_GP)
8778	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8779
8780	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8781	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8782	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8783	fStepping, puIntrState);
8784	}
8785
8786	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8787	uint16_t uGuestIp = pMixedCtx->ip;
8788	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8789	{
8790	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8791	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8792	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8793	}
8794	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8795	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8796
8797	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8798	X86IDTR16 IdtEntry;
8799	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8800	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8801	AssertRCReturn(rc, rc);
8802
8803	/* Construct the stack frame for the interrupt/exception handler. */
8804	VBOXSTRICTRC rcStrict;
8805	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8806	if (rcStrict == VINF_SUCCESS)
8807	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8808	if (rcStrict == VINF_SUCCESS)
8809	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8810
8811	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8812	if (rcStrict == VINF_SUCCESS)
8813	{
8814	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8815	pMixedCtx->rip = IdtEntry.offSel;
8816	pMixedCtx->cs.Sel = IdtEntry.uSel;
8817	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8818	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8819	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8820	&& uVector == X86_XCPT_PF)
8821	pMixedCtx->cr2 = GCPtrFaultAddress;
8822
8823	/* If any other guest-state bits are changed here, make sure to update
8824	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8825	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8826	\| HM_CHANGED_GUEST_RIP
8827	\| HM_CHANGED_GUEST_RFLAGS
8828	\| HM_CHANGED_GUEST_RSP);
8829
8830	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8831	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8832	{
8833	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8834	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8835	Log4(("Clearing inhibition due to STI.\n"));
8836	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8837	}
8838	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8839	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8840
8841	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8842	it, if we are returning to ring-3 before executing guest code. */
8843	pVCpu->hm.s.Event.fPending = false;
8844
8845	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8846	if (fStepping)
8847	rcStrict = VINF_EM_DBG_STEPPED;
8848	}
8849	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8850	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8851	return rcStrict;
8852	}
8853
8854	/*
8855	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8856	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8857	*/
8858	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8859	}
8860
8861	/* Validate. */
8862	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8863	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8864	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8865
8866	/* Inject. */
8867	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8868	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8869	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8870	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8871
8872	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8873	&& uVector == X86_XCPT_PF)
8874	pMixedCtx->cr2 = GCPtrFaultAddress;
8875
8876	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8877	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8878
8879	AssertRCReturn(rc, rc);
8880	return VINF_SUCCESS;
8881	}
8882
8883
8884	/**
8885	* Clears the interrupt-window exiting control in the VMCS and if necessary
8886	* clears the current event in the VMCS as well.
8887	*
8888	* @returns VBox status code.
8889	* @param pVCpu The cross context virtual CPU structure.
8890	*
8891	* @remarks Use this function only to clear events that have not yet been
8892	* delivered to the guest but are injected in the VMCS!
8893	* @remarks No-long-jump zone!!!
8894	*/
8895	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8896	{
8897	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8898
8899	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8900	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8901
8902	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8903	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8904	}
8905
8906
8907	/**
8908	* Enters the VT-x session.
8909	*
8910	* @returns VBox status code.
8911	* @param pVM The cross context VM structure.
8912	* @param pVCpu The cross context virtual CPU structure.
8913	* @param pCpu Pointer to the CPU info struct.
8914	*/
8915	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8916	{
8917	AssertPtr(pVM);
8918	AssertPtr(pVCpu);
8919	Assert(pVM->hm.s.vmx.fSupported);
8920	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8921	NOREF(pCpu); NOREF(pVM);
8922
8923	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8924	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8925
8926	#ifdef VBOX_STRICT
8927	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8928	RTCCUINTREG uHostCR4 = ASMGetCR4();
8929	if (!(uHostCR4 & X86_CR4_VMXE))
8930	{
8931	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8932	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8933	}
8934	#endif
8935
8936	/*
8937	* Load the VCPU's VMCS as the current (and active) one.
8938	*/
8939	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8940	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8941	if (RT_FAILURE(rc))
8942	return rc;
8943
8944	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8945	pVCpu->hm.s.fLeaveDone = false;
8946	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8947
8948	return VINF_SUCCESS;
8949	}
8950
8951
8952	/**
8953	* The thread-context callback (only on platforms which support it).
8954	*
8955	* @param enmEvent The thread-context event.
8956	* @param pVCpu The cross context virtual CPU structure.
8957	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8958	* @thread EMT(pVCpu)
8959	*/
8960	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8961	{
8962	NOREF(fGlobalInit);
8963
8964	switch (enmEvent)
8965	{
8966	case RTTHREADCTXEVENT_OUT:
8967	{
8968	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8969	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8970	VMCPU_ASSERT_EMT(pVCpu);
8971
8972	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8973
8974	/* No longjmps (logger flushes, locks) in this fragile context. */
8975	VMMRZCallRing3Disable(pVCpu);
8976	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8977
8978	/*
8979	* Restore host-state (FPU, debug etc.)
8980	*/
8981	if (!pVCpu->hm.s.fLeaveDone)
8982	{
8983	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8984	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8985	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8986	pVCpu->hm.s.fLeaveDone = true;
8987	}
8988
8989	/* Leave HM context, takes care of local init (term). */
8990	int rc = HMR0LeaveCpu(pVCpu);
8991	AssertRC(rc); NOREF(rc);
8992
8993	/* Restore longjmp state. */
8994	VMMRZCallRing3Enable(pVCpu);
8995	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8996	break;
8997	}
8998
8999	case RTTHREADCTXEVENT_IN:
9000	{
9001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9002	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9003	VMCPU_ASSERT_EMT(pVCpu);
9004
9005	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9006	VMMRZCallRing3Disable(pVCpu);
9007	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9008
9009	/* Initialize the bare minimum state required for HM. This takes care of
9010	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9011	int rc = HMR0EnterCpu(pVCpu);
9012	AssertRC(rc);
9013	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
9014
9015	/* Load the active VMCS as the current one. */
9016	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
9017	{
9018	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
9019	AssertRC(rc); NOREF(rc);
9020	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
9021	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9022	}
9023	pVCpu->hm.s.fLeaveDone = false;
9024
9025	/* Restore longjmp state. */
9026	VMMRZCallRing3Enable(pVCpu);
9027	break;
9028	}
9029
9030	default:
9031	break;
9032	}
9033	}
9034
9035
9036	/**
9037	* Saves the host state in the VMCS host-state.
9038	* Sets up the VM-exit MSR-load area.
9039	*
9040	* The CPU state will be loaded from these fields on every successful VM-exit.
9041	*
9042	* @returns VBox status code.
9043	* @param pVM The cross context VM structure.
9044	* @param pVCpu The cross context virtual CPU structure.
9045	*
9046	* @remarks No-long-jump zone!!!
9047	*/
9048	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
9049	{
9050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9051
9052	int rc = VINF_SUCCESS;
9053	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9054	{
9055	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
9056	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9057
9058	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
9059	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9060
9061	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
9062	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9063
9064	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
9065	}
9066	return rc;
9067	}
9068
9069
9070	/**
9071	* Saves the host state in the VMCS host-state.
9072	*
9073	* @returns VBox status code.
9074	* @param pVM The cross context VM structure.
9075	* @param pVCpu The cross context virtual CPU structure.
9076	*
9077	* @remarks No-long-jump zone!!!
9078	*/
9079	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
9080	{
9081	AssertPtr(pVM);
9082	AssertPtr(pVCpu);
9083
9084	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
9085
9086	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
9087	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
9088	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9089	return hmR0VmxSaveHostState(pVM, pVCpu);
9090	}
9091
9092
9093	/**
9094	* Loads the guest state into the VMCS guest-state area.
9095	*
9096	* The will typically be done before VM-entry when the guest-CPU state and the
9097	* VMCS state may potentially be out of sync.
9098	*
9099	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9100	* VM-entry controls.
9101	* Sets up the appropriate VMX non-root function to execute guest code based on
9102	* the guest CPU mode.
9103	*
9104	* @returns VBox strict status code.
9105	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9106	* without unrestricted guest access and the VMMDev is not presently
9107	* mapped (e.g. EFI32).
9108	*
9109	* @param pVM The cross context VM structure.
9110	* @param pVCpu The cross context virtual CPU structure.
9111	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9112	* out-of-sync. Make sure to update the required fields
9113	* before using them.
9114	*
9115	* @remarks No-long-jump zone!!!
9116	*/
9117	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
9118	{
9119	AssertPtr(pVM);
9120	AssertPtr(pVCpu);
9121	AssertPtr(pMixedCtx);
9122	HMVMX_ASSERT_PREEMPT_SAFE();
9123
9124	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
9125
9126	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
9127
9128	/* Determine real-on-v86 mode. */
9129	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
9130	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
9131	&& CPUMIsGuestInRealModeEx(pMixedCtx))
9132	{
9133	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
9134	}
9135
9136	/*
9137	* Load the guest-state into the VMCS.
9138	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9139	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9140	*/
9141	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
9142	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9143
9144	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
9145	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
9146	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9147
9148	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
9149	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
9150	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9151
9152	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
9153	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9154
9155	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
9156	if (rcStrict == VINF_SUCCESS)
9157	{ /* likely */ }
9158	else
9159	{
9160	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9161	return rcStrict;
9162	}
9163
9164	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
9165	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
9166	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9167
9168	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
9169	determine we don't have to swap EFER after all. */
9170	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
9171	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9172
9173	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
9174	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9175
9176	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
9177	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9178
9179	/*
9180	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
9181	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
9182	*/
9183	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
9184	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
9185
9186	/* Clear any unused and reserved bits. */
9187	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2
9188	\| HM_CHANGED_GUEST_HWVIRT);
9189
9190	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
9191	return rc;
9192	}
9193
9194
9195	/**
9196	* Loads the state shared between the host and guest into the VMCS.
9197	*
9198	* @param pVM The cross context VM structure.
9199	* @param pVCpu The cross context virtual CPU structure.
9200	* @param pCtx Pointer to the guest-CPU context.
9201	*
9202	* @remarks No-long-jump zone!!!
9203	*/
9204	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9205	{
9206	NOREF(pVM);
9207
9208	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9209	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9210
9211	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
9212	{
9213	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
9214	AssertRC(rc);
9215	}
9216
9217	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
9218	{
9219	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
9220	AssertRC(rc);
9221
9222	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9223	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
9224	{
9225	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
9226	AssertRC(rc);
9227	}
9228	}
9229
9230	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS))
9231	{
9232	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
9233	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
9234	}
9235
9236	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
9237	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
9238	{
9239	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
9240	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
9241	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9242	AssertRC(rc);
9243	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
9244	}
9245
9246	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
9247	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9248	}
9249
9250
9251	/**
9252	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9253	*
9254	* @returns Strict VBox status code (i.e. informational status codes too).
9255	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9256	* without unrestricted guest access and the VMMDev is not presently
9257	* mapped (e.g. EFI32).
9258	*
9259	* @param pVM The cross context VM structure.
9260	* @param pVCpu The cross context virtual CPU structure.
9261	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9262	* out-of-sync. Make sure to update the required fields
9263	* before using them.
9264	*
9265	* @remarks No-long-jump zone!!!
9266	*/
9267	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
9268	{
9269	HMVMX_ASSERT_PREEMPT_SAFE();
9270	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9271	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9272
9273	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9274	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9275	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
9276	#endif
9277
9278	/*
9279	* RIP is what changes the most often and hence if it's the only bit needing to be
9280	* updated, we shall handle it early for performance reasons.
9281	*/
9282	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9283	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
9284	{
9285	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
9286	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9287	{ /* likely */}
9288	else
9289	{
9290	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
9291	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9292	}
9293	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
9294	}
9295	else if (HMCPU_CF_VALUE(pVCpu))
9296	{
9297	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
9298	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9299	{ /* likely */}
9300	else
9301	{
9302	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
9303	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9304	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9305	return rcStrict;
9306	}
9307	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
9308	}
9309
9310	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9311	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
9312	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
9313	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9314	return rcStrict;
9315	}
9316
9317
9318	/**
9319	* Does the preparations before executing guest code in VT-x.
9320	*
9321	* This may cause longjmps to ring-3 and may even result in rescheduling to the
9322	* recompiler/IEM. We must be cautious what we do here regarding committing
9323	* guest-state information into the VMCS assuming we assuredly execute the
9324	* guest in VT-x mode.
9325	*
9326	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
9327	* the common-state (TRPM/forceflags), we must undo those changes so that the
9328	* recompiler/IEM can (and should) use them when it resumes guest execution.
9329	* Otherwise such operations must be done when we can no longer exit to ring-3.
9330	*
9331	* @returns Strict VBox status code (i.e. informational status codes too).
9332	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
9333	* have been disabled.
9334	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
9335	* double-fault into the guest.
9336	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
9337	* dispatched directly.
9338	* @retval VINF_* scheduling changes, we have to go back to ring-3.
9339	*
9340	* @param pVM The cross context VM structure.
9341	* @param pVCpu The cross context virtual CPU structure.
9342	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9343	* out-of-sync. Make sure to update the required fields
9344	* before using them.
9345	* @param pVmxTransient Pointer to the VMX transient structure.
9346	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
9347	* us ignore some of the reasons for returning to
9348	* ring-3, and return VINF_EM_DBG_STEPPED if event
9349	* dispatching took place.
9350	*/
9351	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
9352	{
9353	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9354
9355	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
9356	PGMRZDynMapFlushAutoSet(pVCpu);
9357	#endif
9358
9359	/* Check force flag actions that might require us to go back to ring-3. */
9360	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
9361	if (rcStrict == VINF_SUCCESS)
9362	{ /* FFs doesn't get set all the time. */ }
9363	else
9364	return rcStrict;
9365
9366	/*
9367	* Setup the virtualized-APIC accesses.
9368	*
9369	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
9370	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
9371	*
9372	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
9373	*/
9374	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
9375	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
9376	&& PDMHasApic(pVM))
9377	{
9378	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9379	Assert(u64MsrApicBase);
9380	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
9381
9382	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9383
9384	/* Unalias any existing mapping. */
9385	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9386	AssertRCReturn(rc, rc);
9387
9388	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9389	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
9390	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9391	AssertRCReturn(rc, rc);
9392
9393	/* Update the per-VCPU cache of the APIC base MSR. */
9394	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
9395	}
9396
9397	if (TRPMHasTrap(pVCpu))
9398	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
9399	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
9400
9401	/*
9402	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
9403	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
9404	*/
9405	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
9406	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9407	{ /* likely */ }
9408	else
9409	{
9410	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9411	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9412	return rcStrict;
9413	}
9414
9415	/*
9416	* No longjmps to ring-3 from this point on!!!
9417	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
9418	* This also disables flushing of the R0-logger instance (if any).
9419	*/
9420	VMMRZCallRing3Disable(pVCpu);
9421
9422	/*
9423	* Load the guest state bits.
9424	*
9425	* We cannot perform longjmps while loading the guest state because we do not preserve the
9426	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
9427	* CPU migration.
9428	*
9429	* If we are injecting events to a real-on-v86 mode guest, we will have to update
9430	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
9431	* Hence, loading of the guest state needs to be done -after- injection of events.
9432	*/
9433	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
9434	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9435	{ /* likely */ }
9436	else
9437	{
9438	VMMRZCallRing3Enable(pVCpu);
9439	return rcStrict;
9440	}
9441
9442	/*
9443	* We disable interrupts so that we don't miss any interrupts that would flag
9444	* preemption (IPI/timers etc.) when thread-context hooks aren't used and we've
9445	* been running with preemption disabled for a while. Since this is purly to aid
9446	* the RTThreadPreemptIsPending code, it doesn't matter that it may temporarily
9447	* reenable and disable interrupt on NT.
9448	*
9449	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
9450	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
9451	*
9452	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
9453	* executing guest code.
9454	*/
9455	pVmxTransient->fEFlags = ASMIntDisableFlags();
9456
9457	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
9458	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9459	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9460	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
9461	{
9462	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9463	{
9464	pVCpu->hm.s.Event.fPending = false;
9465
9466	/*
9467	* We've injected any pending events. This is really the point of no return (to ring-3).
9468	*
9469	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
9470	* returns from this function, so don't enable them here.
9471	*/
9472	return VINF_SUCCESS;
9473	}
9474
9475	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9476	rcStrict = VINF_EM_RAW_INTERRUPT;
9477	}
9478	else
9479	{
9480	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9481	rcStrict = VINF_EM_RAW_TO_R3;
9482	}
9483
9484	ASMSetFlags(pVmxTransient->fEFlags);
9485	VMMRZCallRing3Enable(pVCpu);
9486
9487	return rcStrict;
9488	}
9489
9490
9491	/**
9492	* Prepares to run guest code in VT-x and we've committed to doing so. This
9493	* means there is no backing out to ring-3 or anywhere else at this
9494	* point.
9495	*
9496	* @param pVM The cross context VM structure.
9497	* @param pVCpu The cross context virtual CPU structure.
9498	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9499	* out-of-sync. Make sure to update the required fields
9500	* before using them.
9501	* @param pVmxTransient Pointer to the VMX transient structure.
9502	*
9503	* @remarks Called with preemption disabled.
9504	* @remarks No-long-jump zone!!!
9505	*/
9506	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9507	{
9508	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9509	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9510	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9511
9512	/*
9513	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9514	*/
9515	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9516	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9517
9518	if (!CPUMIsGuestFPUStateActive(pVCpu))
9519	{
9520	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9521	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9522	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9523	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9524	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9525	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9526	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9527	}
9528
9529	/*
9530	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9531	*/
9532	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9533	&& pVCpu->hm.s.vmx.cMsrs > 0)
9534	{
9535	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9536	}
9537
9538	/*
9539	* Load the host state bits as we may've been preempted (only happens when
9540	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9541	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9542	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9543	* See @bugref{8432}.
9544	*/
9545	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9546	{
9547	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9548	AssertRC(rc);
9549	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9550	}
9551	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9552
9553	/*
9554	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9555	*/
9556	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9557	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9558	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9559
9560	/* Store status of the shared guest-host state at the time of VM-entry. */
9561	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9562	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9563	{
9564	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9565	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9566	}
9567	else
9568	#endif
9569	{
9570	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9571	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9572	}
9573
9574	/*
9575	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9576	*/
9577	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9578	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9579
9580	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9581	RTCPUID idCurrentCpu = pCpu->idCpu;
9582	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9583	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9584	{
9585	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9586	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9587	}
9588
9589	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9590	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9591	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9592	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9593
9594	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9595
9596	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9597	to start executing. */
9598
9599	/*
9600	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9601	*/
9602	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9603	{
9604	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9605	{
9606	bool fMsrUpdated;
9607	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9608	AssertRC(rc2);
9609	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9610
9611	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9612	&fMsrUpdated);
9613	AssertRC(rc2);
9614	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9615
9616	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9617	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9618	}
9619	else
9620	{
9621	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9622	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9623	}
9624	}
9625
9626	if (pVM->cpum.ro.GuestFeatures.fIbrs)
9627	{
9628	bool fMsrUpdated;
9629	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9630	AssertRC(rc2);
9631	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9632
9633	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
9634	&fMsrUpdated);
9635	AssertRC(rc2);
9636	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9637	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9638	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9639	}
9640
9641	#ifdef VBOX_STRICT
9642	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9643	hmR0VmxCheckHostEferMsr(pVCpu);
9644	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9645	#endif
9646	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9647	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9648	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9649	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9650	#endif
9651	}
9652
9653
9654	/**
9655	* Performs some essential restoration of state after running guest code in
9656	* VT-x.
9657	*
9658	* @param pVM The cross context VM structure.
9659	* @param pVCpu The cross context virtual CPU structure.
9660	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9661	* out-of-sync. Make sure to update the required fields
9662	* before using them.
9663	* @param pVmxTransient Pointer to the VMX transient structure.
9664	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9665	*
9666	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9667	*
9668	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9669	* unconditionally when it is safe to do so.
9670	*/
9671	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9672	{
9673	NOREF(pVM);
9674	uint64_t uHostTsc = ASMReadTSC();
9675
9676	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9677
9678	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9679	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9680	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9681	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9682	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9683	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9684
9685	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9686	TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TSCOffset);
9687
9688	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9689	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9690	Assert(!ASMIntAreEnabled());
9691	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9692
9693	#if HC_ARCH_BITS == 64
9694	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9695	#endif
9696	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9697	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9698	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9699	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9700	#else
9701	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9702	#endif
9703	#ifdef VBOX_STRICT
9704	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9705	#endif
9706	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9707
9708	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9709	uint32_t uExitReason;
9710	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9711	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9712	AssertRC(rc);
9713	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9714	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9715
9716	if (rcVMRun == VINF_SUCCESS)
9717	{
9718	/*
9719	* Update the VM-exit history array here even if the VM-entry failed due to:
9720	* - Invalid guest state.
9721	* - MSR loading.
9722	* - Machine-check event.
9723	*
9724	* In any of the above cases we will still have a "valid" VM-exit reason
9725	* despite @a fVMEntryFailed being false.
9726	*
9727	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9728	*
9729	* Note! We don't have CS or RIP at this point. Will probably address that later
9730	* by amending the history entry added here.
9731	*/
9732	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
9733	UINT64_MAX, uHostTsc);
9734
9735	if (!pVmxTransient->fVMEntryFailed)
9736	{
9737	/** @todo We can optimize this by only syncing with our force-flags when
9738	* really needed and keeping the VMCS state as it is for most
9739	* VM-exits. */
9740	/* Update the guest interruptibility-state from the VMCS. */
9741	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9742
9743	/*
9744	* Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9745	* as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9746	* on the longjmp path to ring-3 while saving the (rest of) the guest state,
9747	* see @bugref{6208#c63}.
9748	*/
9749	VMMRZCallRing3Enable(pVCpu);
9750
9751	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9752	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9753	AssertRC(rc);
9754	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9755	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9756	AssertRC(rc);
9757	#endif
9758
9759	/*
9760	* Sync the TPR shadow with our APIC state.
9761	*/
9762	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9763	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9764	{
9765	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9766	AssertRC(rc);
9767	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
9768	}
9769
9770	return;
9771	}
9772	}
9773	else
9774	{
9775	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9776	pVmxTransient->fVMEntryFailed));
9777	}
9778
9779	VMMRZCallRing3Enable(pVCpu);
9780	}
9781
9782
9783	/**
9784	* Runs the guest code using VT-x the normal way.
9785	*
9786	* @returns VBox status code.
9787	* @param pVM The cross context VM structure.
9788	* @param pVCpu The cross context virtual CPU structure.
9789	* @param pCtx Pointer to the guest-CPU context.
9790	*
9791	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9792	*/
9793	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9794	{
9795	VMXTRANSIENT VmxTransient;
9796	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9797	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9798	uint32_t cLoops = 0;
9799
9800	for (;; cLoops++)
9801	{
9802	Assert(!HMR0SuspendPending());
9803	HMVMX_ASSERT_CPU_SAFE();
9804
9805	/* Preparatory work for running guest code, this may force us to return
9806	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9807	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9808	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9809	if (rcStrict != VINF_SUCCESS)
9810	break;
9811
9812	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9813	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9814	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9815
9816	/* Restore any residual host-state and save any bits shared between host
9817	and guest into the guest-CPU state. Re-enables interrupts! */
9818	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9819
9820	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9821	if (RT_SUCCESS(rcRun))
9822	{ /* very likely */ }
9823	else
9824	{
9825	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9826	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9827	return rcRun;
9828	}
9829
9830	/* Profile the VM-exit. */
9831	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9832	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9833	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9834	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9835	HMVMX_START_EXIT_DISPATCH_PROF();
9836
9837	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9838
9839	/* Handle the VM-exit. */
9840	#ifdef HMVMX_USE_FUNCTION_TABLE
9841	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9842	#else
9843	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9844	#endif
9845	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9846	if (rcStrict == VINF_SUCCESS)
9847	{
9848	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9849	continue; /* likely */
9850	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9851	rcStrict = VINF_EM_RAW_INTERRUPT;
9852	}
9853	break;
9854	}
9855
9856	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9857	return rcStrict;
9858	}
9859
9860
9861
9862	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9863	* probes.
9864	*
9865	* The following few functions and associated structure contains the bloat
9866	* necessary for providing detailed debug events and dtrace probes as well as
9867	* reliable host side single stepping. This works on the principle of
9868	* "subclassing" the normal execution loop and workers. We replace the loop
9869	* method completely and override selected helpers to add necessary adjustments
9870	* to their core operation.
9871	*
9872	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9873	* any performance for debug and analysis features.
9874	*
9875	* @{
9876	*/
9877
9878	/**
9879	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9880	* the debug run loop.
9881	*/
9882	typedef struct VMXRUNDBGSTATE
9883	{
9884	/** The RIP we started executing at. This is for detecting that we stepped. */
9885	uint64_t uRipStart;
9886	/** The CS we started executing with. */
9887	uint16_t uCsStart;
9888
9889	/** Whether we've actually modified the 1st execution control field. */
9890	bool fModifiedProcCtls : 1;
9891	/** Whether we've actually modified the 2nd execution control field. */
9892	bool fModifiedProcCtls2 : 1;
9893	/** Whether we've actually modified the exception bitmap. */
9894	bool fModifiedXcptBitmap : 1;
9895
9896	/** We desire the modified the CR0 mask to be cleared. */
9897	bool fClearCr0Mask : 1;
9898	/** We desire the modified the CR4 mask to be cleared. */
9899	bool fClearCr4Mask : 1;
9900	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9901	uint32_t fCpe1Extra;
9902	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9903	uint32_t fCpe1Unwanted;
9904	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9905	uint32_t fCpe2Extra;
9906	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9907	uint32_t bmXcptExtra;
9908	/** The sequence number of the Dtrace provider settings the state was
9909	* configured against. */
9910	uint32_t uDtraceSettingsSeqNo;
9911	/** VM-exits to check (one bit per VM-exit). */
9912	uint32_t bmExitsToCheck[3];
9913
9914	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9915	uint32_t fProcCtlsInitial;
9916	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9917	uint32_t fProcCtls2Initial;
9918	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9919	uint32_t bmXcptInitial;
9920	} VMXRUNDBGSTATE;
9921	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9922	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9923
9924
9925	/**
9926	* Initializes the VMXRUNDBGSTATE structure.
9927	*
9928	* @param pVCpu The cross context virtual CPU structure of the
9929	* calling EMT.
9930	* @param pCtx The CPU register context to go with @a pVCpu.
9931	* @param pDbgState The structure to initialize.
9932	*/
9933	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9934	{
9935	pDbgState->uRipStart = pCtx->rip;
9936	pDbgState->uCsStart = pCtx->cs.Sel;
9937
9938	pDbgState->fModifiedProcCtls = false;
9939	pDbgState->fModifiedProcCtls2 = false;
9940	pDbgState->fModifiedXcptBitmap = false;
9941	pDbgState->fClearCr0Mask = false;
9942	pDbgState->fClearCr4Mask = false;
9943	pDbgState->fCpe1Extra = 0;
9944	pDbgState->fCpe1Unwanted = 0;
9945	pDbgState->fCpe2Extra = 0;
9946	pDbgState->bmXcptExtra = 0;
9947	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9948	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9949	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9950	}
9951
9952
9953	/**
9954	* Updates the VMSC fields with changes requested by @a pDbgState.
9955	*
9956	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9957	* immediately before executing guest code, i.e. when interrupts are disabled.
9958	* We don't check status codes here as we cannot easily assert or return in the
9959	* latter case.
9960	*
9961	* @param pVCpu The cross context virtual CPU structure.
9962	* @param pDbgState The debug state.
9963	*/
9964	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9965	{
9966	/*
9967	* Ensure desired flags in VMCS control fields are set.
9968	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9969	*
9970	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9971	* there should be no stale data in pCtx at this point.
9972	*/
9973	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9974	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9975	{
9976	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9977	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9978	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9979	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9980	pDbgState->fModifiedProcCtls = true;
9981	}
9982
9983	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9984	{
9985	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9986	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9987	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9988	pDbgState->fModifiedProcCtls2 = true;
9989	}
9990
9991	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9992	{
9993	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9994	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9995	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9996	pDbgState->fModifiedXcptBitmap = true;
9997	}
9998
9999	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
10000	{
10001	pVCpu->hm.s.vmx.u32CR0Mask = 0;
10002	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
10003	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
10004	}
10005
10006	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
10007	{
10008	pVCpu->hm.s.vmx.u32CR4Mask = 0;
10009	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
10010	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
10011	}
10012	}
10013
10014
10015	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
10016	{
10017	/*
10018	* Restore VM-exit control settings as we may not reenter this function the
10019	* next time around.
10020	*/
10021	/* We reload the initial value, trigger what we can of recalculations the
10022	next time around. From the looks of things, that's all that's required atm. */
10023	if (pDbgState->fModifiedProcCtls)
10024	{
10025	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
10026	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
10027	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
10028	AssertRCReturn(rc2, rc2);
10029	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
10030	}
10031
10032	/* We're currently the only ones messing with this one, so just restore the
10033	cached value and reload the field. */
10034	if ( pDbgState->fModifiedProcCtls2
10035	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
10036	{
10037	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
10038	AssertRCReturn(rc2, rc2);
10039	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
10040	}
10041
10042	/* If we've modified the exception bitmap, we restore it and trigger
10043	reloading and partial recalculation the next time around. */
10044	if (pDbgState->fModifiedXcptBitmap)
10045	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
10046
10047	return rcStrict;
10048	}
10049
10050
10051	/**
10052	* Configures VM-exit controls for current DBGF and DTrace settings.
10053	*
10054	* This updates @a pDbgState and the VMCS execution control fields to reflect
10055	* the necessary VM-exits demanded by DBGF and DTrace.
10056	*
10057	* @param pVM The cross context VM structure.
10058	* @param pVCpu The cross context virtual CPU structure.
10059	* @param pCtx Pointer to the guest-CPU context.
10060	* @param pDbgState The debug state.
10061	* @param pVmxTransient Pointer to the VMX transient structure. May update
10062	* fUpdateTscOffsettingAndPreemptTimer.
10063	*/
10064	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
10065	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
10066	{
10067	/*
10068	* Take down the dtrace serial number so we can spot changes.
10069	*/
10070	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
10071	ASMCompilerBarrier();
10072
10073	/*
10074	* We'll rebuild most of the middle block of data members (holding the
10075	* current settings) as we go along here, so start by clearing it all.
10076	*/
10077	pDbgState->bmXcptExtra = 0;
10078	pDbgState->fCpe1Extra = 0;
10079	pDbgState->fCpe1Unwanted = 0;
10080	pDbgState->fCpe2Extra = 0;
10081	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
10082	pDbgState->bmExitsToCheck[i] = 0;
10083
10084	/*
10085	* Software interrupts (INT XXh) - no idea how to trigger these...
10086	*/
10087	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
10088	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
10089	{
10090	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
10091	}
10092
10093	/*
10094	* INT3 breakpoints - triggered by #BP exceptions.
10095	*/
10096	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
10097	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
10098
10099	/*
10100	* Exception bitmap and XCPT events+probes.
10101	*/
10102	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
10103	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
10104	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
10105
10106	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
10107	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
10108	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
10109	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
10110	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
10111	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
10112	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
10113	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
10114	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
10115	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
10116	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
10117	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
10118	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
10119	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
10120	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
10121	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
10122	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
10123	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
10124
10125	if (pDbgState->bmXcptExtra)
10126	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
10127
10128	/*
10129	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
10130	*
10131	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
10132	* So, when adding/changing/removing please don't forget to update it.
10133	*
10134	* Some of the macros are picking up local variables to save horizontal space,
10135	* (being able to see it in a table is the lesser evil here).
10136	*/
10137	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
10138	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
10139	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
10140	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
10141	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
10142	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
10143	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
10144	} else do { } while (0)
10145	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
10146	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
10147	{ \
10148	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
10149	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
10150	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
10151	} else do { } while (0)
10152	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
10153	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
10154	{ \
10155	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
10156	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
10157	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
10158	} else do { } while (0)
10159	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
10160	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
10161	{ \
10162	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
10163	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
10164	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
10165	} else do { } while (0)
10166
10167	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
10168	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
10169	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
10170	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
10171	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
10172
10173	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
10174	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
10175	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
10176	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
10177	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
10178	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
10179	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
10180	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
10181	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
10182	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
10183	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
10184	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
10185	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
10186	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
10187	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
10188	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
10189	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
10190	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
10191	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
10192	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
10193	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
10194	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
10195	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
10196	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
10197	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
10198	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
10199	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
10200	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
10201	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
10202	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
10203	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
10204	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
10205	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
10206	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
10207	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
10208	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
10209
10210	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
10211	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
10212	{
10213	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
10214	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
10215	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
10216	AssertRC(rc2);
10217
10218	#if 0 /** @todo fix me */
10219	pDbgState->fClearCr0Mask = true;
10220	pDbgState->fClearCr4Mask = true;
10221	#endif
10222	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
10223	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
10224	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
10225	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
10226	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
10227	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
10228	require clearing here and in the loop if we start using it. */
10229	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
10230	}
10231	else
10232	{
10233	if (pDbgState->fClearCr0Mask)
10234	{
10235	pDbgState->fClearCr0Mask = false;
10236	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10237	}
10238	if (pDbgState->fClearCr4Mask)
10239	{
10240	pDbgState->fClearCr4Mask = false;
10241	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10242	}
10243	}
10244	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
10245	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
10246
10247	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
10248	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
10249	{
10250	/** @todo later, need to fix handler as it assumes this won't usually happen. */
10251	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
10252	}
10253	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
10254	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
10255
10256	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
10257	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
10258	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
10259	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
10260	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
10261	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
10262	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
10263	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
10264	#if 0 /** @todo too slow, fix handler. */
10265	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
10266	#endif
10267	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
10268
10269	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
10270	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
10271	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
10272	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
10273	{
10274	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
10275	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
10276	}
10277	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
10278	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
10279	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
10280	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
10281
10282	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
10283	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
10284	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
10285	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
10286	{
10287	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
10288	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
10289	}
10290	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
10291	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
10292	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
10293	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
10294
10295	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
10296	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
10297	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
10298	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
10299	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
10300	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
10301	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
10302	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
10303	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
10304	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
10305	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
10306	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
10307	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
10308	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
10309	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
10310	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
10311	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
10312	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
10313	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
10314	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
10315	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
10316	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
10317
10318	#undef IS_EITHER_ENABLED
10319	#undef SET_ONLY_XBM_IF_EITHER_EN
10320	#undef SET_CPE1_XBM_IF_EITHER_EN
10321	#undef SET_CPEU_XBM_IF_EITHER_EN
10322	#undef SET_CPE2_XBM_IF_EITHER_EN
10323
10324	/*
10325	* Sanitize the control stuff.
10326	*/
10327	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
10328	if (pDbgState->fCpe2Extra)
10329	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
10330	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
10331	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
10332	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
10333	{
10334	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
10335	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10336	}
10337
10338	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
10339	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
10340	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
10341	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
10342	}
10343
10344
10345	/**
10346	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
10347	* appropriate.
10348	*
10349	* The caller has checked the VM-exit against the
10350	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
10351	* already, so we don't have to do that either.
10352	*
10353	* @returns Strict VBox status code (i.e. informational status codes too).
10354	* @param pVM The cross context VM structure.
10355	* @param pVCpu The cross context virtual CPU structure.
10356	* @param pMixedCtx Pointer to the guest-CPU context.
10357	* @param pVmxTransient Pointer to the VMX-transient structure.
10358	* @param uExitReason The VM-exit reason.
10359	*
10360	* @remarks The name of this function is displayed by dtrace, so keep it short
10361	* and to the point. No longer than 33 chars long, please.
10362	*/
10363	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
10364	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
10365	{
10366	/*
10367	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
10368	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
10369	*
10370	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
10371	* does. Must add/change/remove both places. Same ordering, please.
10372	*
10373	* Added/removed events must also be reflected in the next section
10374	* where we dispatch dtrace events.
10375	*/
10376	bool fDtrace1 = false;
10377	bool fDtrace2 = false;
10378	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
10379	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
10380	uint32_t uEventArg = 0;
10381	#define SET_EXIT(a_EventSubName) \
10382	do { \
10383	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
10384	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
10385	} while (0)
10386	#define SET_BOTH(a_EventSubName) \
10387	do { \
10388	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
10389	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
10390	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
10391	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
10392	} while (0)
10393	switch (uExitReason)
10394	{
10395	case VMX_EXIT_MTF:
10396	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10397
10398	case VMX_EXIT_XCPT_OR_NMI:
10399	{
10400	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
10401	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
10402	{
10403	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10404	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
10405	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
10406	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
10407	{
10408	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
10409	{
10410	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10411	uEventArg = pVmxTransient->uExitIntErrorCode;
10412	}
10413	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
10414	switch (enmEvent1)
10415	{
10416	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
10417	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
10418	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
10419	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
10420	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
10421	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
10422	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
10423	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
10424	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
10425	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
10426	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
10427	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
10428	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
10429	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
10430	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
10431	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
10432	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
10433	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
10434	default: break;
10435	}
10436	}
10437	else
10438	AssertFailed();
10439	break;
10440
10441	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
10442	uEventArg = idxVector;
10443	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
10444	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
10445	break;
10446	}
10447	break;
10448	}
10449
10450	case VMX_EXIT_TRIPLE_FAULT:
10451	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
10452	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
10453	break;
10454	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
10455	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
10456	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
10457	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
10458	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
10459
10460	/* Instruction specific VM-exits: */
10461	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10462	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10463	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10464	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10465	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10466	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10467	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10468	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10469	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10470	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10471	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10472	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10473	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10474	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10475	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10476	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10477	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10478	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10479	case VMX_EXIT_MOV_CRX:
10480	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10481	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10482	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10483	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10484	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10485	SET_BOTH(CRX_READ);
10486	else
10487	SET_BOTH(CRX_WRITE);
10488	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10489	break;
10490	case VMX_EXIT_MOV_DRX:
10491	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10492	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10493	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10494	SET_BOTH(DRX_READ);
10495	else
10496	SET_BOTH(DRX_WRITE);
10497	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10498	break;
10499	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10500	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10501	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10502	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10503	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10504	case VMX_EXIT_XDTR_ACCESS:
10505	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10506	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10507	{
10508	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10509	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10510	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10511	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10512	}
10513	break;
10514
10515	case VMX_EXIT_TR_ACCESS:
10516	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10517	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10518	{
10519	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10520	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10521	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10522	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10523	}
10524	break;
10525
10526	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10527	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10528	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10529	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10530	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10531	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10532	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10533	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10534	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10535	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10536	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10537
10538	/* Events that aren't relevant at this point. */
10539	case VMX_EXIT_EXT_INT:
10540	case VMX_EXIT_INT_WINDOW:
10541	case VMX_EXIT_NMI_WINDOW:
10542	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10543	case VMX_EXIT_PREEMPT_TIMER:
10544	case VMX_EXIT_IO_INSTR:
10545	break;
10546
10547	/* Errors and unexpected events. */
10548	case VMX_EXIT_INIT_SIGNAL:
10549	case VMX_EXIT_SIPI:
10550	case VMX_EXIT_IO_SMI:
10551	case VMX_EXIT_SMI:
10552	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10553	case VMX_EXIT_ERR_MSR_LOAD:
10554	case VMX_EXIT_ERR_MACHINE_CHECK:
10555	break;
10556
10557	default:
10558	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10559	break;
10560	}
10561	#undef SET_BOTH
10562	#undef SET_EXIT
10563
10564	/*
10565	* Dtrace tracepoints go first. We do them here at once so we don't
10566	* have to copy the guest state saving and stuff a few dozen times.
10567	* Down side is that we've got to repeat the switch, though this time
10568	* we use enmEvent since the probes are a subset of what DBGF does.
10569	*/
10570	if (fDtrace1 \|\| fDtrace2)
10571	{
10572	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10573	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10574	switch (enmEvent1)
10575	{
10576	/** @todo consider which extra parameters would be helpful for each probe. */
10577	case DBGFEVENT_END: break;
10578	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10579	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10580	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10581	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10582	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10583	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10584	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10585	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10586	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10587	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10588	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10589	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10590	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10591	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10592	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10593	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10594	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10595	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10596	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10597	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10598	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10599	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10600	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10601	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10602	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10603	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10604	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10605	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10606	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10607	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10608	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10609	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10610	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10611	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10612	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10613	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10614	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10615	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10616	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10617	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10618	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10619	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10620	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10621	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10622	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10623	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10624	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10625	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10626	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10627	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10628	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10629	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10630	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10631	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10632	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10633	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10634	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10635	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10636	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10637	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10638	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10639	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10640	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10641	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10642	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10643	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10644	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10645	}
10646	switch (enmEvent2)
10647	{
10648	/** @todo consider which extra parameters would be helpful for each probe. */
10649	case DBGFEVENT_END: break;
10650	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10651	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10652	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10653	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10654	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10655	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10656	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10657	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10658	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10659	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10660	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10661	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10662	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10663	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10664	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10665	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10666	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10667	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10668	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10669	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10670	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10671	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10672	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10673	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10674	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10675	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10676	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10677	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10678	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10679	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10680	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10681	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10682	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10683	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10684	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10685	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10686	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10687	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10688	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10689	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10690	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10691	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10692	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10693	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10694	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10695	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10696	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10697	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10698	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10699	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10700	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10701	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10702	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10703	}
10704	}
10705
10706	/*
10707	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10708	* the DBGF call will do a full check).
10709	*
10710	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10711	* Note! If we have to events, we prioritize the first, i.e. the instruction
10712	* one, in order to avoid event nesting.
10713	*/
10714	if ( enmEvent1 != DBGFEVENT_END
10715	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10716	{
10717	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10718	if (rcStrict != VINF_SUCCESS)
10719	return rcStrict;
10720	}
10721	else if ( enmEvent2 != DBGFEVENT_END
10722	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10723	{
10724	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10725	if (rcStrict != VINF_SUCCESS)
10726	return rcStrict;
10727	}
10728
10729	return VINF_SUCCESS;
10730	}
10731
10732
10733	/**
10734	* Single-stepping VM-exit filtering.
10735	*
10736	* This is preprocessing the VM-exits and deciding whether we've gotten far
10737	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10738	* handling is performed.
10739	*
10740	* @returns Strict VBox status code (i.e. informational status codes too).
10741	* @param pVM The cross context VM structure.
10742	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10743	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10744	* out-of-sync. Make sure to update the required
10745	* fields before using them.
10746	* @param pVmxTransient Pointer to the VMX-transient structure.
10747	* @param uExitReason The VM-exit reason.
10748	* @param pDbgState The debug state.
10749	*/
10750	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10751	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10752	{
10753	/*
10754	* Expensive (saves context) generic dtrace VM-exit probe.
10755	*/
10756	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10757	{ /* more likely */ }
10758	else
10759	{
10760	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10761	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10762	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10763	}
10764
10765	/*
10766	* Check for host NMI, just to get that out of the way.
10767	*/
10768	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10769	{ /* normally likely */ }
10770	else
10771	{
10772	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10773	AssertRCReturn(rc2, rc2);
10774	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10775	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10776	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10777	}
10778
10779	/*
10780	* Check for single stepping event if we're stepping.
10781	*/
10782	if (pVCpu->hm.s.fSingleInstruction)
10783	{
10784	switch (uExitReason)
10785	{
10786	case VMX_EXIT_MTF:
10787	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10788
10789	/* Various events: */
10790	case VMX_EXIT_XCPT_OR_NMI:
10791	case VMX_EXIT_EXT_INT:
10792	case VMX_EXIT_TRIPLE_FAULT:
10793	case VMX_EXIT_INT_WINDOW:
10794	case VMX_EXIT_NMI_WINDOW:
10795	case VMX_EXIT_TASK_SWITCH:
10796	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10797	case VMX_EXIT_APIC_ACCESS:
10798	case VMX_EXIT_EPT_VIOLATION:
10799	case VMX_EXIT_EPT_MISCONFIG:
10800	case VMX_EXIT_PREEMPT_TIMER:
10801
10802	/* Instruction specific VM-exits: */
10803	case VMX_EXIT_CPUID:
10804	case VMX_EXIT_GETSEC:
10805	case VMX_EXIT_HLT:
10806	case VMX_EXIT_INVD:
10807	case VMX_EXIT_INVLPG:
10808	case VMX_EXIT_RDPMC:
10809	case VMX_EXIT_RDTSC:
10810	case VMX_EXIT_RSM:
10811	case VMX_EXIT_VMCALL:
10812	case VMX_EXIT_VMCLEAR:
10813	case VMX_EXIT_VMLAUNCH:
10814	case VMX_EXIT_VMPTRLD:
10815	case VMX_EXIT_VMPTRST:
10816	case VMX_EXIT_VMREAD:
10817	case VMX_EXIT_VMRESUME:
10818	case VMX_EXIT_VMWRITE:
10819	case VMX_EXIT_VMXOFF:
10820	case VMX_EXIT_VMXON:
10821	case VMX_EXIT_MOV_CRX:
10822	case VMX_EXIT_MOV_DRX:
10823	case VMX_EXIT_IO_INSTR:
10824	case VMX_EXIT_RDMSR:
10825	case VMX_EXIT_WRMSR:
10826	case VMX_EXIT_MWAIT:
10827	case VMX_EXIT_MONITOR:
10828	case VMX_EXIT_PAUSE:
10829	case VMX_EXIT_XDTR_ACCESS:
10830	case VMX_EXIT_TR_ACCESS:
10831	case VMX_EXIT_INVEPT:
10832	case VMX_EXIT_RDTSCP:
10833	case VMX_EXIT_INVVPID:
10834	case VMX_EXIT_WBINVD:
10835	case VMX_EXIT_XSETBV:
10836	case VMX_EXIT_RDRAND:
10837	case VMX_EXIT_INVPCID:
10838	case VMX_EXIT_VMFUNC:
10839	case VMX_EXIT_RDSEED:
10840	case VMX_EXIT_XSAVES:
10841	case VMX_EXIT_XRSTORS:
10842	{
10843	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10844	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10845	AssertRCReturn(rc2, rc2);
10846	if ( pMixedCtx->rip != pDbgState->uRipStart
10847	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10848	return VINF_EM_DBG_STEPPED;
10849	break;
10850	}
10851
10852	/* Errors and unexpected events: */
10853	case VMX_EXIT_INIT_SIGNAL:
10854	case VMX_EXIT_SIPI:
10855	case VMX_EXIT_IO_SMI:
10856	case VMX_EXIT_SMI:
10857	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10858	case VMX_EXIT_ERR_MSR_LOAD:
10859	case VMX_EXIT_ERR_MACHINE_CHECK:
10860	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10861	break;
10862
10863	default:
10864	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10865	break;
10866	}
10867	}
10868
10869	/*
10870	* Check for debugger event breakpoints and dtrace probes.
10871	*/
10872	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10873	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10874	{
10875	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10876	if (rcStrict != VINF_SUCCESS)
10877	return rcStrict;
10878	}
10879
10880	/*
10881	* Normal processing.
10882	*/
10883	#ifdef HMVMX_USE_FUNCTION_TABLE
10884	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10885	#else
10886	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10887	#endif
10888	}
10889
10890
10891	/**
10892	* Single steps guest code using VT-x.
10893	*
10894	* @returns Strict VBox status code (i.e. informational status codes too).
10895	* @param pVM The cross context VM structure.
10896	* @param pVCpu The cross context virtual CPU structure.
10897	* @param pCtx Pointer to the guest-CPU context.
10898	*
10899	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10900	*/
10901	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10902	{
10903	VMXTRANSIENT VmxTransient;
10904	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10905
10906	/* Set HMCPU indicators. */
10907	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10908	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10909	pVCpu->hm.s.fDebugWantRdTscExit = false;
10910	pVCpu->hm.s.fUsingDebugLoop = true;
10911
10912	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10913	VMXRUNDBGSTATE DbgState;
10914	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10915	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10916
10917	/*
10918	* The loop.
10919	*/
10920	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10921	for (uint32_t cLoops = 0; ; cLoops++)
10922	{
10923	Assert(!HMR0SuspendPending());
10924	HMVMX_ASSERT_CPU_SAFE();
10925	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10926
10927	/*
10928	* Preparatory work for running guest code, this may force us to return
10929	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10930	*/
10931	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10932	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10933	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10934	if (rcStrict != VINF_SUCCESS)
10935	break;
10936
10937	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10938	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10939
10940	/*
10941	* Now we can run the guest code.
10942	*/
10943	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10944
10945	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10946
10947	/*
10948	* Restore any residual host-state and save any bits shared between host
10949	* and guest into the guest-CPU state. Re-enables interrupts!
10950	*/
10951	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10952
10953	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10954	if (RT_SUCCESS(rcRun))
10955	{ /* very likely */ }
10956	else
10957	{
10958	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10959	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10960	return rcRun;
10961	}
10962
10963	/* Profile the VM-exit. */
10964	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10965	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10966	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10967	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10968	HMVMX_START_EXIT_DISPATCH_PROF();
10969
10970	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10971
10972	/*
10973	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10974	*/
10975	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10976	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10977	if (rcStrict != VINF_SUCCESS)
10978	break;
10979	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10980	{
10981	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10982	rcStrict = VINF_EM_RAW_INTERRUPT;
10983	break;
10984	}
10985
10986	/*
10987	* Stepping: Did the RIP change, if so, consider it a single step.
10988	* Otherwise, make sure one of the TFs gets set.
10989	*/
10990	if (fStepping)
10991	{
10992	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10993	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10994	AssertRCReturn(rc2, rc2);
10995	if ( pCtx->rip != DbgState.uRipStart
10996	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10997	{
10998	rcStrict = VINF_EM_DBG_STEPPED;
10999	break;
11000	}
11001	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11002	}
11003
11004	/*
11005	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
11006	*/
11007	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
11008	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
11009	}
11010
11011	/*
11012	* Clear the X86_EFL_TF if necessary.
11013	*/
11014	if (pVCpu->hm.s.fClearTrapFlag)
11015	{
11016	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
11017	AssertRCReturn(rc2, rc2);
11018	pVCpu->hm.s.fClearTrapFlag = false;
11019	pCtx->eflags.Bits.u1TF = 0;
11020	}
11021	/** @todo there seems to be issues with the resume flag when the monitor trap
11022	* flag is pending without being used. Seen early in bios init when
11023	* accessing APIC page in protected mode. */
11024
11025	/*
11026	* Restore VM-exit control settings as we may not reenter this function the
11027	* next time around.
11028	*/
11029	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
11030
11031	/* Restore HMCPU indicators. */
11032	pVCpu->hm.s.fUsingDebugLoop = false;
11033	pVCpu->hm.s.fDebugWantRdTscExit = false;
11034	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
11035
11036	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11037	return rcStrict;
11038	}
11039
11040
11041	/** @} */
11042
11043
11044	/**
11045	* Checks if any expensive dtrace probes are enabled and we should go to the
11046	* debug loop.
11047	*
11048	* @returns true if we should use debug loop, false if not.
11049	*/
11050	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
11051	{
11052	/* It's probably faster to OR the raw 32-bit counter variables together.
11053	Since the variables are in an array and the probes are next to one
11054	another (more or less), we have good locality. So, better read
11055	eight-nine cache lines ever time and only have one conditional, than
11056	128+ conditionals, right? */
11057	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
11058	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
11059	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
11060	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
11061	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
11062	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
11063	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
11064	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
11065	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
11066	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
11067	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
11068	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
11069	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
11070	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
11071	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
11072	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
11073	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
11074	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
11075	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
11076	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
11077	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
11078	) != 0
11079	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
11080	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
11081	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
11082	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
11083	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
11084	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
11085	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
11086	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
11087	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
11088	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
11089	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
11090	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
11091	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
11092	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
11093	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
11094	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
11095	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
11096	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
11097	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
11098	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
11099	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
11100	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
11101	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
11102	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
11103	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
11104	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
11105	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
11106	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
11107	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
11108	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
11109	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
11110	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
11111	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
11112	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
11113	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
11114	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
11115	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
11116	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
11117	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
11118	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
11119	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
11120	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
11121	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
11122	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
11123	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
11124	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
11125	) != 0
11126	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
11127	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
11128	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
11129	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
11130	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
11131	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
11132	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
11133	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
11134	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
11135	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
11136	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
11137	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
11138	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
11139	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
11140	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
11141	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
11142	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
11143	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
11144	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
11145	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
11146	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
11147	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
11148	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
11149	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
11150	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
11151	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
11152	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
11153	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
11154	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
11155	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
11156	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
11157	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
11158	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
11159	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
11160	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
11161	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
11162	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
11163	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
11164	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
11165	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
11166	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
11167	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
11168	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
11169	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
11170	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
11171	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
11172	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
11173	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
11174	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
11175	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
11176	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
11177	) != 0;
11178	}
11179
11180
11181	/**
11182	* Runs the guest code using VT-x.
11183	*
11184	* @returns Strict VBox status code (i.e. informational status codes too).
11185	* @param pVM The cross context VM structure.
11186	* @param pVCpu The cross context virtual CPU structure.
11187	* @param pCtx Pointer to the guest-CPU context.
11188	*/
11189	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
11190	{
11191	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11192	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
11193	HMVMX_ASSERT_PREEMPT_SAFE();
11194
11195	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
11196
11197	VBOXSTRICTRC rcStrict;
11198	if ( !pVCpu->hm.s.fUseDebugLoop
11199	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
11200	&& !DBGFIsStepping(pVCpu)
11201	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
11202	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
11203	else
11204	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
11205
11206	if (rcStrict == VERR_EM_INTERPRETER)
11207	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11208	else if (rcStrict == VINF_EM_RESET)
11209	rcStrict = VINF_EM_TRIPLE_FAULT;
11210
11211	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
11212	if (RT_FAILURE(rc2))
11213	{
11214	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
11215	rcStrict = rc2;
11216	}
11217	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
11218	return rcStrict;
11219	}
11220
11221
11222	#ifndef HMVMX_USE_FUNCTION_TABLE
11223	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
11224	{
11225	# ifdef DEBUG_ramshankar
11226	# define VMEXIT_CALL_RET(a_CallExpr) \
11227	do { \
11228	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
11229	VBOXSTRICTRC rcStrict = a_CallExpr; \
11230	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
11231	return rcStrict; \
11232	} while (0)
11233	# else
11234	# define VMEXIT_CALL_RET(a_CallExpr) return a_CallExpr
11235	# endif
11236	switch (rcReason)
11237	{
11238	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
11239	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
11240	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
11241	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
11242	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
11243	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
11244	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
11245	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
11246	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
11247	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
11248	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
11249	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
11250	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
11251	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
11252	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
11253	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
11254	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
11255	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
11256	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
11257	case VMX_EXIT_HLT: VMEXIT_CALL_RET(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
11258	case VMX_EXIT_INVD: VMEXIT_CALL_RET(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
11259	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
11260	case VMX_EXIT_RSM: VMEXIT_CALL_RET(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
11261	case VMX_EXIT_MTF: VMEXIT_CALL_RET(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
11262	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
11263	case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
11264	case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
11265	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
11266	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
11267	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
11268	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
11269	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
11270	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
11271	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
11272
11273	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
11274	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
11275	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
11276	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
11277	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
11278	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
11279	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
11280	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
11281	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
11282
11283	case VMX_EXIT_VMCLEAR:
11284	case VMX_EXIT_VMLAUNCH:
11285	case VMX_EXIT_VMPTRLD:
11286	case VMX_EXIT_VMPTRST:
11287	case VMX_EXIT_VMREAD:
11288	case VMX_EXIT_VMRESUME:
11289	case VMX_EXIT_VMWRITE:
11290	case VMX_EXIT_VMXOFF:
11291	case VMX_EXIT_VMXON:
11292	case VMX_EXIT_INVEPT:
11293	case VMX_EXIT_INVVPID:
11294	case VMX_EXIT_VMFUNC:
11295	case VMX_EXIT_XSAVES:
11296	case VMX_EXIT_XRSTORS:
11297	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
11298	case VMX_EXIT_ENCLS:
11299	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
11300	case VMX_EXIT_PML_FULL:
11301	default:
11302	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
11303	}
11304	#undef VMEXIT_CALL_RET
11305	}
11306	#endif /* !HMVMX_USE_FUNCTION_TABLE */
11307
11308
11309	#ifdef VBOX_STRICT
11310	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
11311	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
11312	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
11313
11314	# define HMVMX_ASSERT_PREEMPT_CPUID() \
11315	do { \
11316	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
11317	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
11318	} while (0)
11319
11320	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
11321	do { \
11322	AssertPtr(pVCpu); \
11323	AssertPtr(pMixedCtx); \
11324	AssertPtr(pVmxTransient); \
11325	Assert(pVmxTransient->fVMEntryFailed == false); \
11326	Assert(ASMIntAreEnabled()); \
11327	HMVMX_ASSERT_PREEMPT_SAFE(); \
11328	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
11329	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
11330	HMVMX_ASSERT_PREEMPT_SAFE(); \
11331	if (VMMR0IsLogFlushDisabled(pVCpu)) \
11332	HMVMX_ASSERT_PREEMPT_CPUID(); \
11333	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
11334	} while (0)
11335
11336	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
11337	do { \
11338	Log4Func(("\n")); \
11339	} while (0)
11340	#else /* nonstrict builds: */
11341	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
11342	do { \
11343	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
11344	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
11345	} while (0)
11346	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
11347	#endif
11348
11349
11350	/**
11351	* Advances the guest RIP by the specified number of bytes.
11352	*
11353	* @param pVCpu The cross context virtual CPU structure.
11354	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
11355	* out-of-sync. Make sure to update the required fields
11356	* before using them.
11357	* @param cbInstr Number of bytes to advance the RIP by.
11358	*
11359	* @remarks No-long-jump zone!!!
11360	*/
11361	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
11362	{
11363	/* Advance the RIP. */
11364	pMixedCtx->rip += cbInstr;
11365	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11366
11367	/* Update interrupt inhibition. */
11368	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
11369	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
11370	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11371	}
11372
11373
11374	/**
11375	* Advances the guest RIP after reading it from the VMCS.
11376	*
11377	* @returns VBox status code, no informational status codes.
11378	* @param pVCpu The cross context virtual CPU structure.
11379	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
11380	* out-of-sync. Make sure to update the required fields
11381	* before using them.
11382	* @param pVmxTransient Pointer to the VMX transient structure.
11383	*
11384	* @remarks No-long-jump zone!!!
11385	*/
11386	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11387	{
11388	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11389	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11390	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11391	AssertRCReturn(rc, rc);
11392
11393	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
11394
11395	/*
11396	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
11397	* pending debug exception field as it takes care of priority of events.
11398	*
11399	* See Intel spec. 32.2.1 "Debug Exceptions".
11400	*/
11401	if ( !pVCpu->hm.s.fSingleInstruction
11402	&& pMixedCtx->eflags.Bits.u1TF)
11403	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11404
11405	return VINF_SUCCESS;
11406	}
11407
11408
11409	/**
11410	* Tries to determine what part of the guest-state VT-x has deemed as invalid
11411	* and update error record fields accordingly.
11412	*
11413	* @return VMX_IGS_* return codes.
11414	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
11415	* wrong with the guest state.
11416	*
11417	* @param pVM The cross context VM structure.
11418	* @param pVCpu The cross context virtual CPU structure.
11419	* @param pCtx Pointer to the guest-CPU state.
11420	*
11421	* @remarks This function assumes our cache of the VMCS controls
11422	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
11423	*/
11424	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
11425	{
11426	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
11427	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
11428	uError = (err); \
11429	break; \
11430	} else do { } while (0)
11431
11432	int rc;
11433	uint32_t uError = VMX_IGS_ERROR;
11434	uint32_t u32Val;
11435	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
11436
11437	do
11438	{
11439	/*
11440	* CR0.
11441	*/
11442	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11443	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11444	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
11445	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
11446	if (fUnrestrictedGuest)
11447	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
11448
11449	uint32_t u32GuestCR0;
11450	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
11451	AssertRCBreak(rc);
11452	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
11453	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
11454	if ( !fUnrestrictedGuest
11455	&& (u32GuestCR0 & X86_CR0_PG)
11456	&& !(u32GuestCR0 & X86_CR0_PE))
11457	{
11458	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
11459	}
11460
11461	/*
11462	* CR4.
11463	*/
11464	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11465	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11466
11467	uint32_t u32GuestCR4;
11468	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11469	AssertRCBreak(rc);
11470	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11471	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11472
11473	/*
11474	* IA32_DEBUGCTL MSR.
11475	*/
11476	uint64_t u64Val;
11477	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11478	AssertRCBreak(rc);
11479	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11480	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11481	{
11482	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11483	}
11484	uint64_t u64DebugCtlMsr = u64Val;
11485
11486	#ifdef VBOX_STRICT
11487	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11488	AssertRCBreak(rc);
11489	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11490	#endif
11491	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11492
11493	/*
11494	* RIP and RFLAGS.
11495	*/
11496	uint32_t u32Eflags;
11497	#if HC_ARCH_BITS == 64
11498	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11499	AssertRCBreak(rc);
11500	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11501	if ( !fLongModeGuest
11502	\|\| !pCtx->cs.Attr.n.u1Long)
11503	{
11504	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11505	}
11506	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11507	* must be identical if the "IA-32e mode guest" VM-entry
11508	* control is 1 and CS.L is 1. No check applies if the
11509	* CPU supports 64 linear-address bits. */
11510
11511	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11512	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11513	AssertRCBreak(rc);
11514	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11515	VMX_IGS_RFLAGS_RESERVED);
11516	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11517	u32Eflags = u64Val;
11518	#else
11519	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11520	AssertRCBreak(rc);
11521	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11522	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11523	#endif
11524
11525	if ( fLongModeGuest
11526	\|\| ( fUnrestrictedGuest
11527	&& !(u32GuestCR0 & X86_CR0_PE)))
11528	{
11529	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11530	}
11531
11532	uint32_t u32EntryInfo;
11533	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11534	AssertRCBreak(rc);
11535	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11536	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11537	{
11538	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11539	}
11540
11541	/*
11542	* 64-bit checks.
11543	*/
11544	#if HC_ARCH_BITS == 64
11545	if (fLongModeGuest)
11546	{
11547	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11548	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11549	}
11550
11551	if ( !fLongModeGuest
11552	&& (u32GuestCR4 & X86_CR4_PCIDE))
11553	{
11554	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11555	}
11556
11557	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11558	* 51:32 beyond the processor's physical-address width are 0. */
11559
11560	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11561	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11562	{
11563	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11564	}
11565
11566	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11567	AssertRCBreak(rc);
11568	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11569
11570	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11571	AssertRCBreak(rc);
11572	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11573	#endif
11574
11575	/*
11576	* PERF_GLOBAL MSR.
11577	*/
11578	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11579	{
11580	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11581	AssertRCBreak(rc);
11582	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11583	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11584	}
11585
11586	/*
11587	* PAT MSR.
11588	*/
11589	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11590	{
11591	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11592	AssertRCBreak(rc);
11593	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11594	for (unsigned i = 0; i < 8; i++)
11595	{
11596	uint8_t u8Val = (u64Val & 0xff);
11597	if ( u8Val != 0 /* UC */
11598	&& u8Val != 1 /* WC */
11599	&& u8Val != 4 /* WT */
11600	&& u8Val != 5 /* WP */
11601	&& u8Val != 6 /* WB */
11602	&& u8Val != 7 /* UC- */)
11603	{
11604	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11605	}
11606	u64Val >>= 8;
11607	}
11608	}
11609
11610	/*
11611	* EFER MSR.
11612	*/
11613	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11614	{
11615	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11616	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11617	AssertRCBreak(rc);
11618	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11619	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11620	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11621	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11622	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11623	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11624	\|\| !(u32GuestCR0 & X86_CR0_PG)
11625	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11626	VMX_IGS_EFER_LMA_LME_MISMATCH);
11627	}
11628
11629	/*
11630	* Segment registers.
11631	*/
11632	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11633	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11634	if (!(u32Eflags & X86_EFL_VM))
11635	{
11636	/* CS */
11637	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11638	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11639	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11640	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11641	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11642	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11643	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11644	/* CS cannot be loaded with NULL in protected mode. */
11645	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11646	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11647	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11648	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11649	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11650	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11651	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11652	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11653	else
11654	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11655
11656	/* SS */
11657	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11658	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11659	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11660	if ( !(pCtx->cr0 & X86_CR0_PE)
11661	\|\| pCtx->cs.Attr.n.u4Type == 3)
11662	{
11663	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11664	}
11665	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11666	{
11667	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11668	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11669	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11670	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11671	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11672	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11673	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11674	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11675	}
11676
11677	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11678	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11679	{
11680	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11681	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11682	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11683	\|\| pCtx->ds.Attr.n.u4Type > 11
11684	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11685	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11686	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11687	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11688	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11689	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11690	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11691	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11692	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11693	}
11694	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11695	{
11696	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11697	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11698	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11699	\|\| pCtx->es.Attr.n.u4Type > 11
11700	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11701	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11702	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11703	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11704	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11705	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11706	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11707	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11708	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11709	}
11710	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11711	{
11712	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11713	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11714	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11715	\|\| pCtx->fs.Attr.n.u4Type > 11
11716	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11717	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11718	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11719	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11720	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11721	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11722	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11723	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11724	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11725	}
11726	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11727	{
11728	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11729	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11730	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11731	\|\| pCtx->gs.Attr.n.u4Type > 11
11732	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11733	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11734	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11735	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11736	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11737	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11738	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11739	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11740	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11741	}
11742	/* 64-bit capable CPUs. */
11743	#if HC_ARCH_BITS == 64
11744	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11745	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11746	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11747	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11748	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11749	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11750	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11751	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11752	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11753	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11754	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11755	#endif
11756	}
11757	else
11758	{
11759	/* V86 mode checks. */
11760	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11761	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11762	{
11763	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11764	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11765	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11766	}
11767	else
11768	{
11769	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11770	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11771	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11772	}
11773
11774	/* CS */
11775	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11776	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11777	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11778	/* SS */
11779	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11780	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11781	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11782	/* DS */
11783	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11784	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11785	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11786	/* ES */
11787	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11788	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11789	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11790	/* FS */
11791	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11792	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11793	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11794	/* GS */
11795	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11796	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11797	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11798	/* 64-bit capable CPUs. */
11799	#if HC_ARCH_BITS == 64
11800	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11801	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11802	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11803	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11804	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11805	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11806	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11807	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11808	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11809	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11810	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11811	#endif
11812	}
11813
11814	/*
11815	* TR.
11816	*/
11817	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11818	/* 64-bit capable CPUs. */
11819	#if HC_ARCH_BITS == 64
11820	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11821	#endif
11822	if (fLongModeGuest)
11823	{
11824	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11825	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11826	}
11827	else
11828	{
11829	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11830	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11831	VMX_IGS_TR_ATTR_TYPE_INVALID);
11832	}
11833	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11834	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11835	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11836	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11837	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11838	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11839	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11840	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11841
11842	/*
11843	* GDTR and IDTR.
11844	*/
11845	#if HC_ARCH_BITS == 64
11846	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11847	AssertRCBreak(rc);
11848	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11849
11850	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11851	AssertRCBreak(rc);
11852	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11853	#endif
11854
11855	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11856	AssertRCBreak(rc);
11857	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11858
11859	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11860	AssertRCBreak(rc);
11861	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11862
11863	/*
11864	* Guest Non-Register State.
11865	*/
11866	/* Activity State. */
11867	uint32_t u32ActivityState;
11868	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11869	AssertRCBreak(rc);
11870	HMVMX_CHECK_BREAK( !u32ActivityState
11871	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11872	VMX_IGS_ACTIVITY_STATE_INVALID);
11873	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11874	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11875	uint32_t u32IntrState;
11876	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11877	AssertRCBreak(rc);
11878	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11879	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11880	{
11881	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11882	}
11883
11884	/** @todo Activity state and injecting interrupts. Left as a todo since we
11885	* currently don't use activity states but ACTIVE. */
11886
11887	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11888	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11889
11890	/* Guest interruptibility-state. */
11891	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11892	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11893	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11894	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11895	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11896	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11897	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11898	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11899	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11900	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11901	{
11902	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11903	{
11904	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11905	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11906	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11907	}
11908	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11909	{
11910	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11911	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11912	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11913	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11914	}
11915	}
11916	/** @todo Assumes the processor is not in SMM. */
11917	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11918	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11919	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11920	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11921	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11922	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11923	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11924	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11925	{
11926	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11927	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11928	}
11929
11930	/* Pending debug exceptions. */
11931	#if HC_ARCH_BITS == 64
11932	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11933	AssertRCBreak(rc);
11934	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11935	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11936	u32Val = u64Val; /* For pending debug exceptions checks below. */
11937	#else
11938	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11939	AssertRCBreak(rc);
11940	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11941	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11942	#endif
11943
11944	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11945	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11946	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11947	{
11948	if ( (u32Eflags & X86_EFL_TF)
11949	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11950	{
11951	/* Bit 14 is PendingDebug.BS. */
11952	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11953	}
11954	if ( !(u32Eflags & X86_EFL_TF)
11955	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11956	{
11957	/* Bit 14 is PendingDebug.BS. */
11958	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11959	}
11960	}
11961
11962	/* VMCS link pointer. */
11963	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11964	AssertRCBreak(rc);
11965	if (u64Val != UINT64_C(0xffffffffffffffff))
11966	{
11967	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11968	/** @todo Bits beyond the processor's physical-address width MBZ. */
11969	/** @todo 32-bit located in memory referenced by value of this field (as a
11970	* physical address) must contain the processor's VMCS revision ID. */
11971	/** @todo SMM checks. */
11972	}
11973
11974	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11975	* not using Nested Paging? */
11976	if ( pVM->hm.s.fNestedPaging
11977	&& !fLongModeGuest
11978	&& CPUMIsGuestInPAEModeEx(pCtx))
11979	{
11980	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11981	AssertRCBreak(rc);
11982	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11983
11984	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11985	AssertRCBreak(rc);
11986	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11987
11988	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11989	AssertRCBreak(rc);
11990	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11991
11992	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11993	AssertRCBreak(rc);
11994	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11995	}
11996
11997	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11998	if (uError == VMX_IGS_ERROR)
11999	uError = VMX_IGS_REASON_NOT_FOUND;
12000	} while (0);
12001
12002	pVCpu->hm.s.u32HMError = uError;
12003	return uError;
12004
12005	#undef HMVMX_ERROR_BREAK
12006	#undef HMVMX_CHECK_BREAK
12007	}
12008
12009	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12010	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12011	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12012
12013	/** @name VM-exit handlers.
12014	* @{
12015	*/
12016
12017	/**
12018	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12019	*/
12020	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12021	{
12022	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12023	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12024	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12025	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12026	return VINF_SUCCESS;
12027	return VINF_EM_RAW_INTERRUPT;
12028	}
12029
12030
12031	/**
12032	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12033	*/
12034	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12035	{
12036	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12037	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12038
12039	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12040	AssertRCReturn(rc, rc);
12041
12042	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
12043	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
12044	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
12045	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12046
12047	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
12048	{
12049	/*
12050	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
12051	* anything we inject is not going to cause a VM-exit directly for the event being injected.
12052	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12053	*
12054	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
12055	*/
12056	VMXDispatchHostNmi();
12057	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12058	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12059	return VINF_SUCCESS;
12060	}
12061
12062	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12063	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12064	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12065	{ /* likely */ }
12066	else
12067	{
12068	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12069	rcStrictRc1 = VINF_SUCCESS;
12070	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12071	return rcStrictRc1;
12072	}
12073
12074	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
12075	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
12076	switch (uIntType)
12077	{
12078	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12079	Assert(uVector == X86_XCPT_DB);
12080	RT_FALL_THRU();
12081	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12082	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
12083	RT_FALL_THRU();
12084	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
12085	{
12086	/*
12087	* If there's any exception caused as a result of event injection, the resulting
12088	* secondary/final execption will be pending, we shall continue guest execution
12089	* after injecting the event. The page-fault case is complicated and we manually
12090	* handle any currently pending event in hmR0VmxExitXcptPF.
12091	*/
12092	if (!pVCpu->hm.s.Event.fPending)
12093	{ /* likely */ }
12094	else if (uVector != X86_XCPT_PF)
12095	{
12096	rc = VINF_SUCCESS;
12097	break;
12098	}
12099
12100	switch (uVector)
12101	{
12102	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
12103	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
12104	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
12105	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
12106	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
12107	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
12108
12109	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12110	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
12111	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12112	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
12113	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12114	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
12115	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12116	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
12117	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12118	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
12119	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12120	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
12121	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12122	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
12123	default:
12124	{
12125	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12126	AssertRCReturn(rc, rc);
12127
12128	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12129	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12130	{
12131	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12132	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12133	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12134
12135	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12136	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12137	AssertRCReturn(rc, rc);
12138	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12139	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12140	0 /* GCPtrFaultAddress */);
12141	AssertRCReturn(rc, rc);
12142	}
12143	else
12144	{
12145	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12146	pVCpu->hm.s.u32HMError = uVector;
12147	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12148	}
12149	break;
12150	}
12151	}
12152	break;
12153	}
12154
12155	default:
12156	{
12157	pVCpu->hm.s.u32HMError = uExitIntInfo;
12158	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12159	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
12160	break;
12161	}
12162	}
12163	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12164	return rc;
12165	}
12166
12167
12168	/**
12169	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12170	*/
12171	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12172	{
12173	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12174
12175	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12176	hmR0VmxClearIntWindowExitVmcs(pVCpu);
12177
12178	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
12179	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12180	return VINF_SUCCESS;
12181	}
12182
12183
12184	/**
12185	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12186	*/
12187	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12188	{
12189	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12190	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
12191	{
12192	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12193	HMVMX_RETURN_UNEXPECTED_EXIT();
12194	}
12195
12196	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
12197
12198	/*
12199	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12200	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12201	*/
12202	uint32_t uIntrState = 0;
12203	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12204	AssertRCReturn(rc, rc);
12205
12206	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
12207	if ( fBlockSti
12208	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12209	{
12210	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12211	}
12212
12213	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12214	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
12215
12216	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
12217	return VINF_SUCCESS;
12218	}
12219
12220
12221	/**
12222	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12223	*/
12224	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12225	{
12226	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
12228	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12229	}
12230
12231
12232	/**
12233	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
12234	*/
12235	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12236	{
12237	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12238	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
12239	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12240	}
12241
12242
12243	/**
12244	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
12245	*/
12246	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12247	{
12248	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12249	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
12250	Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12251
12252	/*
12253	* Get the state we need and update the exit history entry.
12254	*/
12255	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12256	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12257	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12258	rc \|= hmR0VmxSaveGuestCs(pVCpu);
12259	AssertRCReturn(rc, rc);
12260
12261	VBOXSTRICTRC rcStrict;
12262	PCEMEXITREC pExitRec;
12263	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12264	EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM \| EMEXIT_F_PREEMPT_DISABLED,
12265	EMEXITTYPE_CPUID),
12266	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12267	if (!pExitRec)
12268	{
12269	/*
12270	* Regular CPUID instruction execution.
12271	*/
12272	PVM pVM = pVCpu->CTX_SUFF(pVM);
12273	rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12274	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12275	{
12276	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12277	Assert(pVmxTransient->cbInstr == 2);
12278	}
12279	else
12280	{
12281	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
12282	rcStrict = VERR_EM_INTERPRETER;
12283	}
12284	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
12285	}
12286	else
12287	{
12288	/*
12289	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
12290	*/
12291	Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12292	int rc2 = hmR0VmxSaveGuestRegsForIemInterpreting(pVCpu);
12293	AssertRCReturn(rc2, rc2);
12294
12295	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
12296	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
12297
12298	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12299
12300	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12301	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12302	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12303	}
12304	return VBOXSTRICTRC_TODO(rcStrict);
12305	}
12306
12307
12308	/**
12309	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
12310	*/
12311	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12312	{
12313	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12314	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12315	AssertRCReturn(rc, rc);
12316
12317	if (pMixedCtx->cr4 & X86_CR4_SMXE)
12318	return VINF_EM_RAW_EMULATE_INSTR;
12319
12320	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
12321	HMVMX_RETURN_UNEXPECTED_EXIT();
12322	}
12323
12324
12325	/**
12326	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
12327	*/
12328	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12329	{
12330	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12331	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /* Needed for CPL < 0 only, really. */
12332	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12333	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12334	AssertRCReturn(rc, rc);
12335	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
12336	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12337	{
12338	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
12339	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
12340	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12341	}
12342	else if (rcStrict == VINF_IEM_RAISED_XCPT)
12343	rcStrict = VINF_SUCCESS;
12344	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12345	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
12346	return rcStrict;
12347	}
12348
12349
12350	/**
12351	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
12352	*/
12353	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12354	{
12355	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12356	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /* Needed for CPL < 0 only, really. */
12357	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12358	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
12359	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12360	AssertRCReturn(rc, rc);
12361	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
12362	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12363	{
12364	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
12365	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
12366	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12367	}
12368	else if (rcStrict == VINF_IEM_RAISED_XCPT)
12369	rcStrict = VINF_SUCCESS;
12370	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12371	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
12372	return rcStrict;
12373	}
12374
12375
12376	/**
12377	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
12378	*/
12379	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12380	{
12381	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12382	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12383	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12384	AssertRCReturn(rc, rc);
12385
12386	PVM pVM = pVCpu->CTX_SUFF(pVM);
12387	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12388	if (RT_LIKELY(rc == VINF_SUCCESS))
12389	{
12390	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12391	Assert(pVmxTransient->cbInstr == 2);
12392	}
12393	else
12394	{
12395	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
12396	rc = VERR_EM_INTERPRETER;
12397	}
12398	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
12399	return rc;
12400	}
12401
12402
12403	/**
12404	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
12405	*/
12406	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12407	{
12408	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
12410
12411	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
12412	if (EMAreHypercallInstructionsEnabled(pVCpu))
12413	{
12414	#if 0
12415	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12416	#else
12417	/* Aggressive state sync. for now. */
12418	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12419	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
12420	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
12421	AssertRCReturn(rc, rc);
12422	#endif
12423
12424	/* Perform the hypercall. */
12425	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
12426	if (rcStrict == VINF_SUCCESS)
12427	{
12428	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12429	AssertRCReturn(rc, rc);
12430	}
12431	else
12432	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
12433	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
12434	\|\| RT_FAILURE(rcStrict));
12435
12436	/* If the hypercall changes anything other than guest's general-purpose registers,
12437	we would need to reload the guest changed bits here before VM-entry. */
12438	}
12439	else
12440	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
12441
12442	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
12443	if (RT_FAILURE(rcStrict))
12444	{
12445	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12446	rcStrict = VINF_SUCCESS;
12447	}
12448
12449	return rcStrict;
12450	}
12451
12452
12453	/**
12454	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
12455	*/
12456	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12457	{
12458	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12459	PVM pVM = pVCpu->CTX_SUFF(pVM);
12460	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
12461
12462	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12463	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12464	AssertRCReturn(rc, rc);
12465
12466	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
12467	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12468	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12469	else
12470	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
12471	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
12472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
12473	return rcStrict;
12474	}
12475
12476
12477	/**
12478	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
12479	*/
12480	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12481	{
12482	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12483	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12484	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12485	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12486	AssertRCReturn(rc, rc);
12487
12488	PVM pVM = pVCpu->CTX_SUFF(pVM);
12489	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12490	if (RT_LIKELY(rc == VINF_SUCCESS))
12491	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12492	else
12493	{
12494	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
12495	rc = VERR_EM_INTERPRETER;
12496	}
12497	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
12498	return rc;
12499	}
12500
12501
12502	/**
12503	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12504	*/
12505	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12506	{
12507	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12508	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12509	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12510	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12511	AssertRCReturn(rc, rc);
12512
12513	PVM pVM = pVCpu->CTX_SUFF(pVM);
12514	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12515	rc = VBOXSTRICTRC_VAL(rc2);
12516	if (RT_LIKELY( rc == VINF_SUCCESS
12517	\|\| rc == VINF_EM_HALT))
12518	{
12519	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12520	AssertRCReturn(rc3, rc3);
12521
12522	if ( rc == VINF_EM_HALT
12523	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12524	{
12525	rc = VINF_SUCCESS;
12526	}
12527	}
12528	else
12529	{
12530	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12531	rc = VERR_EM_INTERPRETER;
12532	}
12533	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
12534	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12535	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12536	return rc;
12537	}
12538
12539
12540	/**
12541	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12542	*/
12543	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12544	{
12545	/*
12546	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12547	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12548	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
12549	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12550	*/
12551	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12552	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12553	HMVMX_RETURN_UNEXPECTED_EXIT();
12554	}
12555
12556
12557	/**
12558	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12559	*/
12560	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12561	{
12562	/*
12563	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12564	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12565	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12566	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12567	*/
12568	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12569	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12570	HMVMX_RETURN_UNEXPECTED_EXIT();
12571	}
12572
12573
12574	/**
12575	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12576	*/
12577	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12578	{
12579	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12580	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12581	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12582	HMVMX_RETURN_UNEXPECTED_EXIT();
12583	}
12584
12585
12586	/**
12587	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12588	*/
12589	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12590	{
12591	/*
12592	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12593	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12594	* See Intel spec. 25.3 "Other Causes of VM-exits".
12595	*/
12596	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12597	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12598	HMVMX_RETURN_UNEXPECTED_EXIT();
12599	}
12600
12601
12602	/**
12603	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12604	* VM-exit.
12605	*/
12606	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12607	{
12608	/*
12609	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12610	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12611	*
12612	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12613	* See Intel spec. "23.8 Restrictions on VMX operation".
12614	*/
12615	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12616	return VINF_SUCCESS;
12617	}
12618
12619
12620	/**
12621	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12622	* VM-exit.
12623	*/
12624	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12625	{
12626	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12627	return VINF_EM_RESET;
12628	}
12629
12630
12631	/**
12632	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12633	*/
12634	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12635	{
12636	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12637	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12638
12639	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12640	AssertRCReturn(rc, rc);
12641
12642	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12643	rc = VINF_SUCCESS;
12644	else
12645	rc = VINF_EM_HALT;
12646
12647	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12648	if (rc != VINF_SUCCESS)
12649	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12650	return rc;
12651	}
12652
12653
12654	/**
12655	* VM-exit handler for instructions that result in a \#UD exception delivered to
12656	* the guest.
12657	*/
12658	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12659	{
12660	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12661	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12662	return VINF_SUCCESS;
12663	}
12664
12665
12666	/**
12667	* VM-exit handler for expiry of the VMX preemption timer.
12668	*/
12669	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12670	{
12671	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12672
12673	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12674	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12675
12676	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12677	PVM pVM = pVCpu->CTX_SUFF(pVM);
12678	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12679	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12680	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12681	}
12682
12683
12684	/**
12685	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12686	*/
12687	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12688	{
12689	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12690
12691	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12692	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12693	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12694	AssertRCReturn(rc, rc);
12695
12696	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12697	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12698
12699	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12700
12701	return rcStrict;
12702	}
12703
12704
12705	/**
12706	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12707	*/
12708	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12709	{
12710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12711	/** @todo Use VM-exit instruction information. */
12712	return VERR_EM_INTERPRETER;
12713	}
12714
12715
12716	/**
12717	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12718	* Error VM-exit.
12719	*/
12720	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12721	{
12722	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12723	AssertRCReturn(rc, rc);
12724
12725	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12726	AssertRCReturn(rc, rc);
12727
12728	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12729	NOREF(uInvalidReason);
12730
12731	#ifdef VBOX_STRICT
12732	uint32_t uIntrState;
12733	RTHCUINTREG uHCReg;
12734	uint64_t u64Val;
12735	uint32_t u32Val;
12736
12737	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12738	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12739	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12740	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12741	AssertRCReturn(rc, rc);
12742
12743	Log4(("uInvalidReason %u\n", uInvalidReason));
12744	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12745	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12746	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12747	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12748
12749	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12750	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12751	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12752	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12753	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12754	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12755	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12756	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12757	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12758	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12759	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12760	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12761	#else
12762	NOREF(pVmxTransient);
12763	#endif
12764
12765	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12766	return VERR_VMX_INVALID_GUEST_STATE;
12767	}
12768
12769
12770	/**
12771	* VM-exit handler for VM-entry failure due to an MSR-load
12772	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12773	*/
12774	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12775	{
12776	NOREF(pVmxTransient);
12777	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12778	HMVMX_RETURN_UNEXPECTED_EXIT();
12779	}
12780
12781
12782	/**
12783	* VM-exit handler for VM-entry failure due to a machine-check event
12784	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12785	*/
12786	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12787	{
12788	NOREF(pVmxTransient);
12789	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12790	HMVMX_RETURN_UNEXPECTED_EXIT();
12791	}
12792
12793
12794	/**
12795	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12796	* theory.
12797	*/
12798	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12799	{
12800	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12801	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12802	return VERR_VMX_UNDEFINED_EXIT_CODE;
12803	}
12804
12805
12806	/**
12807	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12808	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12809	* Conditional VM-exit.
12810	*/
12811	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12812	{
12813	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12814
12815	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12816	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12817	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12818	return VERR_EM_INTERPRETER;
12819	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12820	HMVMX_RETURN_UNEXPECTED_EXIT();
12821	}
12822
12823
12824	/**
12825	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12826	*/
12827	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12828	{
12829	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12830
12831	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12832	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12833	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12834	return VERR_EM_INTERPRETER;
12835	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12836	HMVMX_RETURN_UNEXPECTED_EXIT();
12837	}
12838
12839
12840	/**
12841	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12842	*/
12843	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12844	{
12845	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12846
12847	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12848	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12849	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12850	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12851	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12852	{
12853	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12854	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12855	}
12856	AssertRCReturn(rc, rc);
12857	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12858
12859	#ifdef VBOX_STRICT
12860	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12861	{
12862	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12863	&& pMixedCtx->ecx != MSR_K6_EFER)
12864	{
12865	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12866	pMixedCtx->ecx));
12867	HMVMX_RETURN_UNEXPECTED_EXIT();
12868	}
12869	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12870	{
12871	VMXMSREXITREAD enmRead;
12872	VMXMSREXITWRITE enmWrite;
12873	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12874	AssertRCReturn(rc2, rc2);
12875	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12876	{
12877	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12878	HMVMX_RETURN_UNEXPECTED_EXIT();
12879	}
12880	}
12881	}
12882	#endif
12883
12884	PVM pVM = pVCpu->CTX_SUFF(pVM);
12885	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12886	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12887	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12888	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12889	if (RT_SUCCESS(rc))
12890	{
12891	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12892	Assert(pVmxTransient->cbInstr == 2);
12893	}
12894	return rc;
12895	}
12896
12897
12898	/**
12899	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12900	*/
12901	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12902	{
12903	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12904	PVM pVM = pVCpu->CTX_SUFF(pVM);
12905	int rc = VINF_SUCCESS;
12906
12907	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12908	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12909	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12910	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12911	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12912	{
12913	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12914	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12915	}
12916	AssertRCReturn(rc, rc);
12917	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12918
12919	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12920	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12921	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12922
12923	if (RT_SUCCESS(rc))
12924	{
12925	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12926
12927	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12928	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12929	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12930	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12931	{
12932	/*
12933	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12934	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12935	* EMInterpretWrmsr() changes it.
12936	*/
12937	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12938	}
12939	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12940	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12941	else if (pMixedCtx->ecx == MSR_K6_EFER)
12942	{
12943	/*
12944	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12945	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12946	* the other bits as well, SCE and NXE. See @bugref{7368}.
12947	*/
12948	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12949	}
12950
12951	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12952	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12953	{
12954	switch (pMixedCtx->ecx)
12955	{
12956	/*
12957	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12958	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12959	*/
12960	case MSR_IA32_SYSENTER_CS:
12961	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12962	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12963	break;
12964	case MSR_IA32_SYSENTER_EIP:
12965	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12966	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12967	break;
12968	case MSR_IA32_SYSENTER_ESP:
12969	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12970	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12971	break;
12972	case MSR_K8_FS_BASE: RT_FALL_THRU();
12973	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12974	case MSR_K6_EFER: /* already handled above */ break;
12975	default:
12976	{
12977	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12978	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12979	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12980	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
12981	break;
12982	}
12983	}
12984	}
12985	#ifdef VBOX_STRICT
12986	else
12987	{
12988	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12989	switch (pMixedCtx->ecx)
12990	{
12991	case MSR_IA32_SYSENTER_CS:
12992	case MSR_IA32_SYSENTER_EIP:
12993	case MSR_IA32_SYSENTER_ESP:
12994	case MSR_K8_FS_BASE:
12995	case MSR_K8_GS_BASE:
12996	{
12997	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12998	HMVMX_RETURN_UNEXPECTED_EXIT();
12999	}
13000
13001	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13002	default:
13003	{
13004	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
13005	{
13006	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
13007	if (pMixedCtx->ecx != MSR_K6_EFER)
13008	{
13009	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13010	pMixedCtx->ecx));
13011	HMVMX_RETURN_UNEXPECTED_EXIT();
13012	}
13013	}
13014
13015	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
13016	{
13017	VMXMSREXITREAD enmRead;
13018	VMXMSREXITWRITE enmWrite;
13019	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
13020	AssertRCReturn(rc2, rc2);
13021	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
13022	{
13023	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
13024	HMVMX_RETURN_UNEXPECTED_EXIT();
13025	}
13026	}
13027	break;
13028	}
13029	}
13030	}
13031	#endif /* VBOX_STRICT */
13032	}
13033	return rc;
13034	}
13035
13036
13037	/**
13038	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13039	*/
13040	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13041	{
13042	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13043
13044	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
13045	return VINF_EM_RAW_INTERRUPT;
13046	}
13047
13048
13049	/**
13050	* VM-exit handler for when the TPR value is lowered below the specified
13051	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13052	*/
13053	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13054	{
13055	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13056	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
13057
13058	/*
13059	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
13060	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
13061	*/
13062	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13063	return VINF_SUCCESS;
13064	}
13065
13066
13067	/**
13068	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13069	* VM-exit.
13070	*
13071	* @retval VINF_SUCCESS when guest execution can continue.
13072	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
13073	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13074	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13075	* interpreter.
13076	*/
13077	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13078	{
13079	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13080	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13081	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13082	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13083	AssertRCReturn(rc, rc);
13084
13085	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
13086	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
13087	PVM pVM = pVCpu->CTX_SUFF(pVM);
13088	VBOXSTRICTRC rcStrict;
13089	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
13090	switch (uAccessType)
13091	{
13092	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
13093	{
13094	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13095	AssertRCReturn(rc, rc);
13096
13097	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
13098	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
13099	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
13100	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
13101	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13102	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
13103	{
13104	case 0: /* CR0 */
13105	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13106	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
13107	break;
13108	case 2: /* CR2 */
13109	/* Nothing to do here, CR2 it's not part of the VMCS. */
13110	break;
13111	case 3: /* CR3 */
13112	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
13113	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
13114	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
13115	break;
13116	case 4: /* CR4 */
13117	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
13118	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
13119	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13120	break;
13121	case 8: /* CR8 */
13122	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
13123	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
13124	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
13125	break;
13126	default:
13127	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
13128	break;
13129	}
13130
13131	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
13132	break;
13133	}
13134
13135	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
13136	{
13137	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13138	AssertRCReturn(rc, rc);
13139
13140	Assert( !pVM->hm.s.fNestedPaging
13141	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
13142	\|\| pVCpu->hm.s.fUsingDebugLoop
13143	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
13144
13145	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13146	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
13147	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
13148
13149	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
13150	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
13151	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
13152	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13153	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
13154	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
13155	VBOXSTRICTRC_VAL(rcStrict)));
13156	if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
13157	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
13158	break;
13159	}
13160
13161	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
13162	{
13163	AssertRCReturn(rc, rc);
13164	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
13165	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13166	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13167	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13168	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
13169	break;
13170	}
13171
13172	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
13173	{
13174	AssertRCReturn(rc, rc);
13175	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
13176	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
13177	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
13178	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13179	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13180	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13181	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
13182	break;
13183	}
13184
13185	default:
13186	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
13187	VERR_VMX_UNEXPECTED_EXCEPTION);
13188	}
13189
13190	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
13191	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
13192	NOREF(pVM);
13193	return rcStrict;
13194	}
13195
13196
13197	/**
13198	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
13199	* VM-exit.
13200	*/
13201	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13202	{
13203	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13204	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
13205	Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
13206
13207	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13208	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13209	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
13210	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
13211	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13212	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13213	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
13214	AssertRCReturn(rc, rc);
13215
13216	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
13217	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
13218	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
13219	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
13220	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
13221	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
13222	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13223	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13224	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
13225
13226	/*
13227	* Update exit history to see if this exit can be optimized.
13228	*/
13229	VBOXSTRICTRC rcStrict;
13230	PCEMEXITREC pExitRec = NULL;
13231	if ( !fGstStepping
13232	&& !fDbgStepping)
13233	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13234	!fIOString
13235	? !fIOWrite
13236	? EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM \| EMEXIT_F_PREEMPT_DISABLED,
13237	EMEXITTYPE_IO_PORT_READ)
13238	: EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM \| EMEXIT_F_PREEMPT_DISABLED,
13239	EMEXITTYPE_IO_PORT_WRITE)
13240	: !fIOWrite
13241	? EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM \| EMEXIT_F_PREEMPT_DISABLED,
13242	EMEXITTYPE_IO_PORT_STR_READ)
13243	: EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM \| EMEXIT_F_PREEMPT_DISABLED,
13244	EMEXITTYPE_IO_PORT_STR_WRITE),
13245	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13246	if (!pExitRec)
13247	{
13248	/* I/O operation lookup arrays. */
13249	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
13250	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
13251
13252	uint32_t const cbValue = s_aIOSizes[uIOWidth];
13253	uint32_t const cbInstr = pVmxTransient->cbInstr;
13254	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
13255	PVM pVM = pVCpu->CTX_SUFF(pVM);
13256	if (fIOString)
13257	{
13258	/*
13259	* INS/OUTS - I/O String instruction.
13260	*
13261	* Use instruction-information if available, otherwise fall back on
13262	* interpreting the instruction.
13263	*/
13264	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
13265	fIOWrite ? 'w' : 'r'));
13266	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
13267	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
13268	{
13269	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13270	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
13271	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13272	AssertRCReturn(rc2, rc2);
13273	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
13274	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
13275	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
13276	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
13277	if (fIOWrite)
13278	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
13279	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
13280	else
13281	{
13282	/*
13283	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
13284	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
13285	* See Intel Instruction spec. for "INS".
13286	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
13287	*/
13288	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
13289	}
13290	}
13291	else
13292	{
13293	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
13294	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13295	AssertRCReturn(rc2, rc2);
13296	rcStrict = IEMExecOne(pVCpu);
13297	}
13298	/** @todo IEM needs to be setting these flags somehow. */
13299	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
13300	fUpdateRipAlready = true;
13301	}
13302	else
13303	{
13304	/*
13305	* IN/OUT - I/O instruction.
13306	*/
13307	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
13308	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
13309	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
13310	if (fIOWrite)
13311	{
13312	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
13313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
13314	}
13315	else
13316	{
13317	uint32_t u32Result = 0;
13318	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
13319	if (IOM_SUCCESS(rcStrict))
13320	{
13321	/* Save result of I/O IN instr. in AL/AX/EAX. */
13322	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
13323	}
13324	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
13325	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
13326	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
13327	}
13328	}
13329
13330	if (IOM_SUCCESS(rcStrict))
13331	{
13332	if (!fUpdateRipAlready)
13333	{
13334	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
13335	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
13336	}
13337
13338	/*
13339	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
13340	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
13341	*/
13342	if (fIOString)
13343	{
13344	/** @todo Single-step for INS/OUTS with REP prefix? */
13345	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13346	}
13347	else if ( !fDbgStepping
13348	&& fGstStepping)
13349	{
13350	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13351	}
13352
13353	/*
13354	* If any I/O breakpoints are armed, we need to check if one triggered
13355	* and take appropriate action.
13356	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
13357	*/
13358	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13359	AssertRCReturn(rc2, rc2);
13360
13361	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
13362	* execution engines about whether hyper BPs and such are pending. */
13363	uint32_t const uDr7 = pMixedCtx->dr[7];
13364	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
13365	&& X86_DR7_ANY_RW_IO(uDr7)
13366	&& (pMixedCtx->cr4 & X86_CR4_DE))
13367	\|\| DBGFBpIsHwIoArmed(pVM)))
13368	{
13369	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
13370
13371	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
13372	VMMRZCallRing3Disable(pVCpu);
13373	HM_DISABLE_PREEMPT();
13374
13375	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
13376
13377	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
13378	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
13379	{
13380	/* Raise #DB. */
13381	if (fIsGuestDbgActive)
13382	ASMSetDR6(pMixedCtx->dr[6]);
13383	if (pMixedCtx->dr[7] != uDr7)
13384	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13385
13386	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
13387	}
13388	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
13389	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
13390	else if ( rcStrict2 != VINF_SUCCESS
13391	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
13392	rcStrict = rcStrict2;
13393	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
13394
13395	HM_RESTORE_PREEMPT();
13396	VMMRZCallRing3Enable(pVCpu);
13397	}
13398	}
13399
13400	#ifdef VBOX_STRICT
13401	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
13402	Assert(!fIOWrite);
13403	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
13404	Assert(fIOWrite);
13405	else
13406	{
13407	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
13408	* statuses, that the VMM device and some others may return. See
13409	* IOM_SUCCESS() for guidance. */
13410	AssertMsg( RT_FAILURE(rcStrict)
13411	\|\| rcStrict == VINF_SUCCESS
13412	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
13413	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
13414	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
13415	\|\| rcStrict == VINF_EM_RAW_TO_R3
13416	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13417	# endif
13418	}
13419	#endif
13420	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
13421	}
13422	else
13423	{
13424	/*
13425	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13426	*/
13427	int rc2 = hmR0VmxSaveGuestRegsForIemInterpreting(pVCpu);
13428	AssertRCReturn(rc2, rc2);
13429
13430	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
13431	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13432	VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
13433	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
13434
13435	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13436
13437	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13438	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13439	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13440	}
13441	return rcStrict;
13442	}
13443
13444
13445	/**
13446	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
13447	* VM-exit.
13448	*/
13449	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13450	{
13451	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13452
13453	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
13454	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13455	AssertRCReturn(rc, rc);
13456	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
13457	{
13458	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13459	AssertRCReturn(rc, rc);
13460	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
13461	{
13462	uint32_t uErrCode;
13463	RTGCUINTPTR GCPtrFaultAddress;
13464	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13465	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13466	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
13467	if (fErrorCodeValid)
13468	{
13469	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13470	AssertRCReturn(rc, rc);
13471	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
13472	}
13473	else
13474	uErrCode = 0;
13475
13476	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13477	&& uVector == X86_XCPT_PF)
13478	GCPtrFaultAddress = pMixedCtx->cr2;
13479	else
13480	GCPtrFaultAddress = 0;
13481
13482	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13483	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
13484
13485	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
13486	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13487	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13488	}
13489	}
13490
13491	/* Fall back to the interpreter to emulate the task-switch. */
13492	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13493	return VERR_EM_INTERPRETER;
13494	}
13495
13496
13497	/**
13498	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
13499	*/
13500	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13501	{
13502	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13503	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
13504	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
13505	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13506	AssertRCReturn(rc, rc);
13507	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
13508	return VINF_EM_DBG_STEPPED;
13509	}
13510
13511
13512	/**
13513	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13514	*/
13515	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13516	{
13517	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13518
13519	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13520
13521	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13522	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13523	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13524	{
13525	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13526	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13527	{
13528	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13529	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13530	}
13531	}
13532	else
13533	{
13534	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13535	rcStrict1 = VINF_SUCCESS;
13536	return rcStrict1;
13537	}
13538
13539	#if 0
13540	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13541	* just sync the whole thing. */
13542	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13543	#else
13544	/* Aggressive state sync. for now. */
13545	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13546	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13547	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13548	#endif
13549	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13550	AssertRCReturn(rc, rc);
13551
13552	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13553	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13554	VBOXSTRICTRC rcStrict2;
13555	switch (uAccessType)
13556	{
13557	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13558	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13559	{
13560	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13561	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13562	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13563
13564	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13565	GCPhys &= PAGE_BASE_GC_MASK;
13566	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13567	PVM pVM = pVCpu->CTX_SUFF(pVM);
13568	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13569	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13570
13571	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13572	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13573	CPUMCTX2CORE(pMixedCtx), GCPhys);
13574	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13575	if ( rcStrict2 == VINF_SUCCESS
13576	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13577	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13578	{
13579	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13580	\| HM_CHANGED_GUEST_RSP
13581	\| HM_CHANGED_GUEST_RFLAGS
13582	\| HM_CHANGED_GUEST_APIC_STATE);
13583	rcStrict2 = VINF_SUCCESS;
13584	}
13585	break;
13586	}
13587
13588	default:
13589	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13590	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13591	break;
13592	}
13593
13594	if (rcStrict2 != VINF_SUCCESS)
13595	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13596	return rcStrict2;
13597	}
13598
13599
13600	/**
13601	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13602	* VM-exit.
13603	*/
13604	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13605	{
13606	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13607
13608	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13609	if (pVmxTransient->fWasGuestDebugStateActive)
13610	{
13611	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13612	HMVMX_RETURN_UNEXPECTED_EXIT();
13613	}
13614
13615	if ( !pVCpu->hm.s.fSingleInstruction
13616	&& !pVmxTransient->fWasHyperDebugStateActive)
13617	{
13618	Assert(!DBGFIsStepping(pVCpu));
13619	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13620
13621	/* Don't intercept MOV DRx any more. */
13622	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13623	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13624	AssertRCReturn(rc, rc);
13625
13626	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13627	VMMRZCallRing3Disable(pVCpu);
13628	HM_DISABLE_PREEMPT();
13629
13630	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13631	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13632	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13633
13634	HM_RESTORE_PREEMPT();
13635	VMMRZCallRing3Enable(pVCpu);
13636
13637	#ifdef VBOX_WITH_STATISTICS
13638	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13639	AssertRCReturn(rc, rc);
13640	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13641	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13642	else
13643	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13644	#endif
13645	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13646	return VINF_SUCCESS;
13647	}
13648
13649	/*
13650	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13651	* Update the segment registers and DR7 from the CPU.
13652	*/
13653	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13654	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13655	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13656	AssertRCReturn(rc, rc);
13657	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13658
13659	PVM pVM = pVCpu->CTX_SUFF(pVM);
13660	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13661	{
13662	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13663	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13664	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13665	if (RT_SUCCESS(rc))
13666	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13668	}
13669	else
13670	{
13671	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13672	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13673	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13674	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13675	}
13676
13677	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13678	if (RT_SUCCESS(rc))
13679	{
13680	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13681	AssertRCReturn(rc2, rc2);
13682	return VINF_SUCCESS;
13683	}
13684	return rc;
13685	}
13686
13687
13688	/**
13689	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13690	* Conditional VM-exit.
13691	*/
13692	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13693	{
13694	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13695	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13696
13697	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13698	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13699	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13700	{
13701	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13702	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13703	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13704	{
13705	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13706	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13707	}
13708	}
13709	else
13710	{
13711	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13712	rcStrict1 = VINF_SUCCESS;
13713	return rcStrict1;
13714	}
13715
13716	/*
13717	* Get sufficent state and update the exit history entry.
13718	*/
13719	RTGCPHYS GCPhys = 0;
13720	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13721
13722	#if 0
13723	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13724	#else
13725	/* Aggressive state sync. for now. */
13726	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13727	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13728	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13729	#endif
13730	AssertRCReturn(rc, rc);
13731
13732	VBOXSTRICTRC rcStrict;
13733	PCEMEXITREC pExitRec;
13734	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13735	EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM \| EMEXIT_F_PREEMPT_DISABLED,
13736	EMEXITTYPE_MMIO),
13737	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13738	if (!pExitRec)
13739	{
13740	/*
13741	* If we succeed, resume guest execution.
13742	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13743	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13744	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13745	* weird case. See @bugref{6043}.
13746	*/
13747	PVM pVM = pVCpu->CTX_SUFF(pVM);
13748	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13749	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
13750	if ( rcStrict == VINF_SUCCESS
13751	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
13752	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
13753	{
13754	/* Successfully handled MMIO operation. */
13755	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13756	\| HM_CHANGED_GUEST_RSP
13757	\| HM_CHANGED_GUEST_RFLAGS
13758	\| HM_CHANGED_GUEST_APIC_STATE);
13759	rcStrict = VINF_SUCCESS;
13760	}
13761	}
13762	else
13763	{
13764	/*
13765	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13766	*/
13767	Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
13768	int rc2 = hmR0VmxSaveGuestRegsForIemInterpreting(pVCpu);
13769	AssertRCReturn(rc2, rc2);
13770
13771	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
13772	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
13773
13774	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13775
13776	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13777	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13778	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13779	}
13780	return VBOXSTRICTRC_TODO(rcStrict);
13781	}
13782
13783
13784	/**
13785	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13786	* VM-exit.
13787	*/
13788	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13789	{
13790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13791	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13792
13793	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13794	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13795	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13796	{
13797	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13798	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13799	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13800	}
13801	else
13802	{
13803	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13804	rcStrict1 = VINF_SUCCESS;
13805	return rcStrict1;
13806	}
13807
13808	RTGCPHYS GCPhys = 0;
13809	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13810	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13811	#if 0
13812	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13813	#else
13814	/* Aggressive state sync. for now. */
13815	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13816	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13817	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13818	#endif
13819	AssertRCReturn(rc, rc);
13820
13821	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13822	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13823
13824	RTGCUINT uErrorCode = 0;
13825	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13826	uErrorCode \|= X86_TRAP_PF_ID;
13827	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13828	uErrorCode \|= X86_TRAP_PF_RW;
13829	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13830	uErrorCode \|= X86_TRAP_PF_P;
13831
13832	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13833
13834	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13835	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13836
13837	/* Handle the pagefault trap for the nested shadow table. */
13838	PVM pVM = pVCpu->CTX_SUFF(pVM);
13839	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13840	TRPMResetTrap(pVCpu);
13841
13842	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13843	if ( rcStrict2 == VINF_SUCCESS
13844	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13845	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13846	{
13847	/* Successfully synced our nested page tables. */
13848	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13849	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13850	\| HM_CHANGED_GUEST_RSP
13851	\| HM_CHANGED_GUEST_RFLAGS);
13852	return VINF_SUCCESS;
13853	}
13854
13855	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13856	return rcStrict2;
13857	}
13858
13859	/** @} */
13860
13861	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13862	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13863	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13864
13865	/** @name VM-exit exception handlers.
13866	* @{
13867	*/
13868
13869	/**
13870	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13871	*/
13872	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13873	{
13874	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13875	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13876
13877	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13878	AssertRCReturn(rc, rc);
13879
13880	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13881	{
13882	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13883	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13884
13885	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13886	* provides VM-exit instruction length. If this causes problem later,
13887	* disassemble the instruction like it's done on AMD-V. */
13888	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13889	AssertRCReturn(rc2, rc2);
13890	return rc;
13891	}
13892
13893	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13894	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13895	return rc;
13896	}
13897
13898
13899	/**
13900	* VM-exit exception handler for \#BP (Breakpoint exception).
13901	*/
13902	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13903	{
13904	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13905	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13906
13907	/** @todo Try optimize this by not saving the entire guest state unless
13908	* really needed. */
13909	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13910	AssertRCReturn(rc, rc);
13911
13912	PVM pVM = pVCpu->CTX_SUFF(pVM);
13913	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13914	if (rc == VINF_EM_RAW_GUEST_TRAP)
13915	{
13916	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13917	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13918	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13919	AssertRCReturn(rc, rc);
13920
13921	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13922	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13923	}
13924
13925	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13926	return rc;
13927	}
13928
13929
13930	/**
13931	* VM-exit exception handler for \#AC (alignment check exception).
13932	*/
13933	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13934	{
13935	RT_NOREF_PV(pMixedCtx);
13936	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13937
13938	/*
13939	* Re-inject it. We'll detect any nesting before getting here.
13940	*/
13941	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13942	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13943	AssertRCReturn(rc, rc);
13944	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13945
13946	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13947	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13948	return VINF_SUCCESS;
13949	}
13950
13951
13952	/**
13953	* VM-exit exception handler for \#DB (Debug exception).
13954	*/
13955	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13956	{
13957	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13958	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13959	Log6(("XcptDB\n"));
13960
13961	/*
13962	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13963	* for processing.
13964	*/
13965	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13966	AssertRCReturn(rc, rc);
13967
13968	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13969	uint64_t uDR6 = X86_DR6_INIT_VAL;
13970	uDR6 \|= ( pVmxTransient->uExitQualification
13971	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13972
13973	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13974	if (rc == VINF_EM_RAW_GUEST_TRAP)
13975	{
13976	/*
13977	* The exception was for the guest. Update DR6, DR7.GD and
13978	* IA32_DEBUGCTL.LBR before forwarding it.
13979	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13980	*/
13981	VMMRZCallRing3Disable(pVCpu);
13982	HM_DISABLE_PREEMPT();
13983
13984	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13985	pMixedCtx->dr[6] \|= uDR6;
13986	if (CPUMIsGuestDebugStateActive(pVCpu))
13987	ASMSetDR6(pMixedCtx->dr[6]);
13988
13989	HM_RESTORE_PREEMPT();
13990	VMMRZCallRing3Enable(pVCpu);
13991
13992	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13993	AssertRCReturn(rc, rc);
13994
13995	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13996	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13997
13998	/* Paranoia. */
13999	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14000	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
14001
14002	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
14003	AssertRCReturn(rc, rc);
14004
14005	/*
14006	* Raise #DB in the guest.
14007	*
14008	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
14009	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP (INT1) and not the
14010	* regular #DB. Thus it -may- trigger different handling in the CPU (like skipped DPL checks), see @bugref{6398}.
14011	*
14012	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of Intel 386,
14013	* see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14014	*/
14015	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14016	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14017	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14018	AssertRCReturn(rc, rc);
14019	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14020	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14021	return VINF_SUCCESS;
14022	}
14023
14024	/*
14025	* Not a guest trap, must be a hypervisor related debug event then.
14026	* Update DR6 in case someone is interested in it.
14027	*/
14028	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14029	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14030	CPUMSetHyperDR6(pVCpu, uDR6);
14031
14032	return rc;
14033	}
14034
14035	/**
14036	* VM-exit exception handler for \#GP (General-protection exception).
14037	*
14038	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14039	*/
14040	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
14041	{
14042	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
14043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14044
14045	int rc;
14046	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
14047	{ /* likely */ }
14048	else
14049	{
14050	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14051	Assert(pVCpu->hm.s.fUsingDebugLoop);
14052	#endif
14053	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
14054	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14055	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14056	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14057	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
14058	AssertRCReturn(rc, rc);
14059	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
14060	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
14061	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14062	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14063	return rc;
14064	}
14065
14066	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
14067	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14068
14069	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
14070	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
14071	AssertRCReturn(rc, rc);
14072
14073	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
14074	uint32_t cbOp = 0;
14075	PVM pVM = pVCpu->CTX_SUFF(pVM);
14076	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14077	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
14078	if (RT_SUCCESS(rc))
14079	{
14080	rc = VINF_SUCCESS;
14081	Assert(cbOp == pDis->cbInstr);
14082	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
14083	switch (pDis->pCurInstr->uOpcode)
14084	{
14085	case OP_CLI:
14086	{
14087	pMixedCtx->eflags.Bits.u1IF = 0;
14088	pMixedCtx->eflags.Bits.u1RF = 0;
14089	pMixedCtx->rip += pDis->cbInstr;
14090	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14091	if ( !fDbgStepping
14092	&& pMixedCtx->eflags.Bits.u1TF)
14093	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
14094	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
14095	break;
14096	}
14097
14098	case OP_STI:
14099	{
14100	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
14101	pMixedCtx->eflags.Bits.u1IF = 1;
14102	pMixedCtx->eflags.Bits.u1RF = 0;
14103	pMixedCtx->rip += pDis->cbInstr;
14104	if (!fOldIF)
14105	{
14106	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
14107	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
14108	}
14109	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14110	if ( !fDbgStepping
14111	&& pMixedCtx->eflags.Bits.u1TF)
14112	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
14113	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
14114	break;
14115	}
14116
14117	case OP_HLT:
14118	{
14119	rc = VINF_EM_HALT;
14120	pMixedCtx->rip += pDis->cbInstr;
14121	pMixedCtx->eflags.Bits.u1RF = 0;
14122	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
14124	break;
14125	}
14126
14127	case OP_POPF:
14128	{
14129	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
14130	uint32_t cbParm;
14131	uint32_t uMask;
14132	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
14133	if (pDis->fPrefix & DISPREFIX_OPSIZE)
14134	{
14135	cbParm = 4;
14136	uMask = 0xffffffff;
14137	}
14138	else
14139	{
14140	cbParm = 2;
14141	uMask = 0xffff;
14142	}
14143
14144	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
14145	RTGCPTR GCPtrStack = 0;
14146	X86EFLAGS Eflags;
14147	Eflags.u32 = 0;
14148	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
14149	&GCPtrStack);
14150	if (RT_SUCCESS(rc))
14151	{
14152	Assert(sizeof(Eflags.u32) >= cbParm);
14153	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
14154	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
14155	}
14156	if (RT_FAILURE(rc))
14157	{
14158	rc = VERR_EM_INTERPRETER;
14159	break;
14160	}
14161	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
14162	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
14163	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
14164	pMixedCtx->esp += cbParm;
14165	pMixedCtx->esp &= uMask;
14166	pMixedCtx->rip += pDis->cbInstr;
14167	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
14168	\| HM_CHANGED_GUEST_RSP
14169	\| HM_CHANGED_GUEST_RFLAGS);
14170	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
14171	POPF restores EFLAGS.TF. */
14172	if ( !fDbgStepping
14173	&& fGstStepping)
14174	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
14175	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
14176	break;
14177	}
14178
14179	case OP_PUSHF:
14180	{
14181	uint32_t cbParm;
14182	uint32_t uMask;
14183	if (pDis->fPrefix & DISPREFIX_OPSIZE)
14184	{
14185	cbParm = 4;
14186	uMask = 0xffffffff;
14187	}
14188	else
14189	{
14190	cbParm = 2;
14191	uMask = 0xffff;
14192	}
14193
14194	/* Get the stack pointer & push the contents of eflags onto the stack. */
14195	RTGCPTR GCPtrStack = 0;
14196	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
14197	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
14198	if (RT_FAILURE(rc))
14199	{
14200	rc = VERR_EM_INTERPRETER;
14201	break;
14202	}
14203	X86EFLAGS Eflags = pMixedCtx->eflags;
14204	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
14205	Eflags.Bits.u1RF = 0;
14206	Eflags.Bits.u1VM = 0;
14207
14208	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
14209	if (RT_UNLIKELY(rc != VINF_SUCCESS))
14210	{
14211	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
14212	rc = VERR_EM_INTERPRETER;
14213	break;
14214	}
14215	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
14216	pMixedCtx->esp -= cbParm;
14217	pMixedCtx->esp &= uMask;
14218	pMixedCtx->rip += pDis->cbInstr;
14219	pMixedCtx->eflags.Bits.u1RF = 0;
14220	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
14221	\| HM_CHANGED_GUEST_RSP
14222	\| HM_CHANGED_GUEST_RFLAGS);
14223	if ( !fDbgStepping
14224	&& pMixedCtx->eflags.Bits.u1TF)
14225	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
14226	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
14227	break;
14228	}
14229
14230	case OP_IRET:
14231	{
14232	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
14233	* instruction reference. */
14234	RTGCPTR GCPtrStack = 0;
14235	uint32_t uMask = 0xffff;
14236	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
14237	uint16_t aIretFrame[3];
14238	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
14239	{
14240	rc = VERR_EM_INTERPRETER;
14241	break;
14242	}
14243	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
14244	&GCPtrStack);
14245	if (RT_SUCCESS(rc))
14246	{
14247	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
14248	PGMACCESSORIGIN_HM));
14249	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
14250	}
14251	if (RT_FAILURE(rc))
14252	{
14253	rc = VERR_EM_INTERPRETER;
14254	break;
14255	}
14256	pMixedCtx->eip = 0;
14257	pMixedCtx->ip = aIretFrame[0];
14258	pMixedCtx->cs.Sel = aIretFrame[1];
14259	pMixedCtx->cs.ValidSel = aIretFrame[1];
14260	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
14261	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
14262	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
14263	pMixedCtx->sp += sizeof(aIretFrame);
14264	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
14265	\| HM_CHANGED_GUEST_SEGMENT_REGS
14266	\| HM_CHANGED_GUEST_RSP
14267	\| HM_CHANGED_GUEST_RFLAGS);
14268	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
14269	if ( !fDbgStepping
14270	&& fGstStepping)
14271	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
14272	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
14273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
14274	break;
14275	}
14276
14277	case OP_INT:
14278	{
14279	uint16_t uVector = pDis->Param1.uValue & 0xff;
14280	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
14281	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
14282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
14283	break;
14284	}
14285
14286	case OP_INTO:
14287	{
14288	if (pMixedCtx->eflags.Bits.u1OF)
14289	{
14290	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
14291	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
14292	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
14293	}
14294	else
14295	{
14296	pMixedCtx->eflags.Bits.u1RF = 0;
14297	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
14298	}
14299	break;
14300	}
14301
14302	default:
14303	{
14304	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
14305	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
14306	EMCODETYPE_SUPERVISOR);
14307	rc = VBOXSTRICTRC_VAL(rc2);
14308	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
14309	/** @todo We have to set pending-debug exceptions here when the guest is
14310	* single-stepping depending on the instruction that was interpreted. */
14311	Log4(("#GP rc=%Rrc\n", rc));
14312	break;
14313	}
14314	}
14315	}
14316	else
14317	rc = VERR_EM_INTERPRETER;
14318
14319	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
14320	("#GP Unexpected rc=%Rrc\n", rc));
14321	return rc;
14322	}
14323
14324
14325	/**
14326	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
14327	* the exception reported in the VMX transient structure back into the VM.
14328	*
14329	* @remarks Requires uExitIntInfo in the VMX transient structure to be
14330	* up-to-date.
14331	*/
14332	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
14333	{
14334	RT_NOREF_PV(pMixedCtx);
14335	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
14336	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14337	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
14338	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
14339	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
14340	#endif
14341
14342	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
14343	hmR0VmxCheckExitDueToEventDelivery(). */
14344	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14345	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14346	AssertRCReturn(rc, rc);
14347	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
14348
14349	#ifdef DEBUG_ramshankar
14350	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
14351	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14352	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14353	#endif
14354
14355	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14356	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14357	return VINF_SUCCESS;
14358	}
14359
14360
14361	/**
14362	* VM-exit exception handler for \#PF (Page-fault exception).
14363	*/
14364	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
14365	{
14366	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
14367	PVM pVM = pVCpu->CTX_SUFF(pVM);
14368	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
14369	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14370	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14371	AssertRCReturn(rc, rc);
14372
14373	if (!pVM->hm.s.fNestedPaging)
14374	{ /* likely */ }
14375	else
14376	{
14377	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
14378	Assert(pVCpu->hm.s.fUsingDebugLoop);
14379	#endif
14380	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
14381	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
14382	{
14383	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14384	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
14385	}
14386	else
14387	{
14388	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14389	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
14390	Log4(("Pending #DF due to vectoring #PF. NP\n"));
14391	}
14392	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14393	return rc;
14394	}
14395
14396	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
14397	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
14398	if (pVmxTransient->fVectoringPF)
14399	{
14400	Assert(pVCpu->hm.s.Event.fPending);
14401	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14402	}
14403
14404	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
14405	AssertRCReturn(rc, rc);
14406
14407	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
14408	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
14409
14410	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
14411	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
14412	(RTGCPTR)pVmxTransient->uExitQualification);
14413
14414	Log4(("#PF: rc=%Rrc\n", rc));
14415	if (rc == VINF_SUCCESS)
14416	{
14417	#if 0
14418	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
14419	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
14420	* memory? We don't update the whole state here... */
14421	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
14422	\| HM_CHANGED_GUEST_RSP
14423	\| HM_CHANGED_GUEST_RFLAGS
14424	\| HM_CHANGED_GUEST_APIC_STATE);
14425	#else
14426	/*
14427	* This is typically a shadow page table sync or a MMIO instruction. But we may have
14428	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
14429	*/
14430	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
14431	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
14432	#endif
14433	TRPMResetTrap(pVCpu);
14434	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14435	return rc;
14436	}
14437
14438	if (rc == VINF_EM_RAW_GUEST_TRAP)
14439	{
14440	if (!pVmxTransient->fVectoringDoublePF)
14441	{
14442	/* It's a guest page fault and needs to be reflected to the guest. */
14443	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14444	TRPMResetTrap(pVCpu);
14445	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14446	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14447	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
14448	}
14449	else
14450	{
14451	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14452	TRPMResetTrap(pVCpu);
14453	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14454	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
14455	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
14456	}
14457
14458	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14459	return VINF_SUCCESS;
14460	}
14461
14462	TRPMResetTrap(pVCpu);
14463	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14464	return rc;
14465	}
14466
14467	/** @} */
14468

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72643

Download in other formats: