HMVMXR0.cpp@ 58944

Last change on this file since 58944 was 58938, checked in by vboxsync, 9 years ago
HM,DBGF: Made DBGF notify HM about changes to VMM event and interrupt breakpoints. Made HM cache the basic info wrt ring-0 loop selection, opting for using a debug loop when debugging takes place to avoid cluttering slowing down the normal execution loop. The plan is to extend the single stepping loop and to put complicated dtrace probes into the same loop. Modified the VMX loop selection already.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 514.2 KB

Line
1	/* $Id: HMVMXR0.cpp 58938 2015-12-01 14:17:45Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#include "HMInternal.h"
38	#include <VBox/vmm/vm.h>
39	#include "HMVMXR0.h"
40	#include "dtrace/VBoxVMM.h"
41
42	#ifdef DEBUG_ramshankar
43	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
44	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
45	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
46	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
47	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
48	# define HMVMX_ALWAYS_TRAP_PF
49	# define HMVMX_ALWAYS_SWAP_FPU_STATE
50	# define HMVMX_ALWAYS_FLUSH_TLB
51	# define HMVMX_ALWAYS_SWAP_EFER
52	#endif
53
54
55	/*********************************************************************************************************************************
56	* Defined Constants And Macros *
57	*********************************************************************************************************************************/
58	/** Use the function table. */
59	#define HMVMX_USE_FUNCTION_TABLE
60
61	/** Determine which tagged-TLB flush handler to use. */
62	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67	/** @name Updated-guest-state flags.
68	* @{ */
69	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
70	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
71	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
72	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
73	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
74	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
75	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
76	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
77	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
78	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
79	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
80	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
81	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
82	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
84	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
85	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
86	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
87	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
88	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
89	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
90	\| HMVMX_UPDATED_GUEST_RSP \
91	\| HMVMX_UPDATED_GUEST_RFLAGS \
92	\| HMVMX_UPDATED_GUEST_CR0 \
93	\| HMVMX_UPDATED_GUEST_CR3 \
94	\| HMVMX_UPDATED_GUEST_CR4 \
95	\| HMVMX_UPDATED_GUEST_GDTR \
96	\| HMVMX_UPDATED_GUEST_IDTR \
97	\| HMVMX_UPDATED_GUEST_LDTR \
98	\| HMVMX_UPDATED_GUEST_TR \
99	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
100	\| HMVMX_UPDATED_GUEST_DEBUG \
101	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
102	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
104	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
105	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
106	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
107	\| HMVMX_UPDATED_GUEST_INTR_STATE \
108	\| HMVMX_UPDATED_GUEST_APIC_STATE)
109	/** @} */
110
111	/** @name
112	* Flags to skip redundant reads of some common VMCS fields that are not part of
113	* the guest-CPU state but are in the transient structure.
114	*/
115	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
122	/** @} */
123
124	/** @name
125	* States of the VMCS.
126	*
127	* This does not reflect all possible VMCS states but currently only those
128	* needed for maintaining the VMCS consistently even when thread-context hooks
129	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
130	*/
131	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
132	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
133	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
134	/** @} */
135
136	/**
137	* Exception bitmap mask for real-mode guests (real-on-v86).
138	*
139	* We need to intercept all exceptions manually except:
140	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
141	* - \#DB handled in hmR0VmxLoadSharedDebugState().
142	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
143	* support.
144	*/
145	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
146	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
147	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
148	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
149	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
150	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
151	\| RT_BIT(X86_XCPT_XF))
152
153	/**
154	* Exception bitmap mask for all contributory exceptions.
155	*
156	* Page fault is deliberately excluded here as it's conditional as to whether
157	* it's contributory or benign. Page faults are handled separately.
158	*/
159	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
160	\| RT_BIT(X86_XCPT_DE))
161
162	/** Maximum VM-instruction error number. */
163	#define HMVMX_INSTR_ERROR_MAX 28
164
165	/** Profiling macro. */
166	#ifdef HM_PROFILE_EXIT_DISPATCH
167	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
168	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
169	#else
170	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
172	#endif
173
174	/** Assert that preemption is disabled or covered by thread-context hooks. */
175	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
176	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
177
178	/** Assert that we haven't migrated CPUs when thread-context hooks are not
179	* used. */
180	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
181	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
182	("Illegal migration! Entered on CPU %u Current %u\n", \
183	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
184
185	/** Helper macro for VM-exit handlers called unexpectedly. */
186	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
187	do { \
188	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
189	return VERR_VMX_UNEXPECTED_EXIT; \
190	} while (0)
191
192
193	/*********************************************************************************************************************************
194	* Structures and Typedefs *
195	*********************************************************************************************************************************/
196	/**
197	* VMX transient state.
198	*
199	* A state structure for holding miscellaneous information across
200	* VMX non-root operation and restored after the transition.
201	*/
202	typedef struct VMXTRANSIENT
203	{
204	/** The host's rflags/eflags. */
205	RTCCUINTREG fEFlags;
206	#if HC_ARCH_BITS == 32
207	uint32_t u32Alignment0;
208	#endif
209	/** The guest's TPR value used for TPR shadowing. */
210	uint8_t u8GuestTpr;
211	/** Alignment. */
212	uint8_t abAlignment0[7];
213
214	/** The basic VM-exit reason. */
215	uint16_t uExitReason;
216	/** Alignment. */
217	uint16_t u16Alignment0;
218	/** The VM-exit interruption error code. */
219	uint32_t uExitIntErrorCode;
220	/** The VM-exit exit code qualification. */
221	uint64_t uExitQualification;
222
223	/** The VM-exit interruption-information field. */
224	uint32_t uExitIntInfo;
225	/** The VM-exit instruction-length field. */
226	uint32_t cbInstr;
227	/** The VM-exit instruction-information field. */
228	union
229	{
230	/** Plain unsigned int representation. */
231	uint32_t u;
232	/** INS and OUTS information. */
233	struct
234	{
235	uint32_t u6Reserved0 : 7;
236	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
237	uint32_t u3AddrSize : 3;
238	uint32_t u5Reserved1 : 5;
239	/** The segment register (X86_SREG_XXX). */
240	uint32_t iSegReg : 3;
241	uint32_t uReserved2 : 14;
242	} StrIo;
243	} ExitInstrInfo;
244	/** Whether the VM-entry failed or not. */
245	bool fVMEntryFailed;
246	/** Alignment. */
247	uint8_t abAlignment1[3];
248
249	/** The VM-entry interruption-information field. */
250	uint32_t uEntryIntInfo;
251	/** The VM-entry exception error code field. */
252	uint32_t uEntryXcptErrorCode;
253	/** The VM-entry instruction length field. */
254	uint32_t cbEntryInstr;
255
256	/** IDT-vectoring information field. */
257	uint32_t uIdtVectoringInfo;
258	/** IDT-vectoring error code. */
259	uint32_t uIdtVectoringErrorCode;
260
261	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
262	uint32_t fVmcsFieldsRead;
263
264	/** Whether the guest FPU was active at the time of VM-exit. */
265	bool fWasGuestFPUStateActive;
266	/** Whether the guest debug state was active at the time of VM-exit. */
267	bool fWasGuestDebugStateActive;
268	/** Whether the hyper debug state was active at the time of VM-exit. */
269	bool fWasHyperDebugStateActive;
270	/** Whether TSC-offsetting should be setup before VM-entry. */
271	bool fUpdateTscOffsettingAndPreemptTimer;
272	/** Whether the VM-exit was caused by a page-fault during delivery of a
273	* contributory exception or a page-fault. */
274	bool fVectoringDoublePF;
275	/** Whether the VM-exit was caused by a page-fault during delivery of an
276	* external interrupt or NMI. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns Strict VBox status code.
319	* @param pVCpu The cross context virtual CPU structure.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333	/**
334	* VMX VM-exit handler, non-strict status code.
335	*
336	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
337	*
338	* @returns VBox status code, no informational status code returned.
339	* @param pVCpu The cross context virtual CPU structure.
340	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
341	* out-of-sync. Make sure to update the required
342	* fields before using them.
343	* @param pVmxTransient Pointer to the VMX-transient structure.
344	*
345	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
346	* use of that status code will be replaced with VINF_EM_SOMETHING
347	* later when switching over to IEM.
348	*/
349	#ifndef HMVMX_USE_FUNCTION_TABLE
350	typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
351	#else
352	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
353	#endif
354
355
356	/*********************************************************************************************************************************
357	* Internal Functions *
358	*********************************************************************************************************************************/
359	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
360	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
361	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
362	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
363	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
364	bool fStepping, uint32_t *puIntState);
365	#if HC_ARCH_BITS == 32
366	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
367	#endif
368	#ifndef HMVMX_USE_FUNCTION_TABLE
369	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
370	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
371	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
372	#else
373	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
374	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
375	#endif
376
377	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
378	uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
379
380	/** @name VM-exit handlers.
381	* @{
382	*/
383	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395	static FNVMXEXITHANDLER hmR0VmxExitHlt;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411	static FNVMXEXITHANDLER hmR0VmxExitMwait;
412	static FNVMXEXITHANDLER hmR0VmxExitMtf;
413	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414	static FNVMXEXITHANDLER hmR0VmxExitPause;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428	/** @} */
429
430	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
438	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439	#endif
440	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443	/*********************************************************************************************************************************
444	* Global Variables *
445	*********************************************************************************************************************************/
446	#ifdef HMVMX_USE_FUNCTION_TABLE
447
448	/**
449	* VMX_EXIT dispatch table.
450	*/
451	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452	{
453	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493	/* 40 UNDEFINED */ hmR0VmxExitPause,
494	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
510	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513	/* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
515	/* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518	};
519	#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521	#ifdef VBOX_STRICT
522	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523	{
524	/* 0 */ "(Not Used)",
525	/* 1 */ "VMCALL executed in VMX root operation.",
526	/* 2 */ "VMCLEAR with invalid physical address.",
527	/* 3 */ "VMCLEAR with VMXON pointer.",
528	/* 4 */ "VMLAUNCH with non-clear VMCS.",
529	/* 5 */ "VMRESUME with non-launched VMCS.",
530	/* 6 */ "VMRESUME after VMXOFF",
531	/* 7 */ "VM-entry with invalid control fields.",
532	/* 8 */ "VM-entry with invalid host state fields.",
533	/* 9 */ "VMPTRLD with invalid physical address.",
534	/* 10 */ "VMPTRLD with VMXON pointer.",
535	/* 11 */ "VMPTRLD with incorrect revision identifier.",
536	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537	/* 13 */ "VMWRITE to read-only VMCS component.",
538	/* 14 */ "(Not Used)",
539	/* 15 */ "VMXON executed in VMX root operation.",
540	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541	/* 17 */ "VM-entry with non-launched executing VMCS.",
542	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543	/* 19 */ "VMCALL with non-clear VMCS.",
544	/* 20 */ "VMCALL with invalid VM-exit control fields.",
545	/* 21 */ "(Not Used)",
546	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548	/* 24 */ "VMCALL with invalid SMM-monitor features.",
549	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550	/* 26 */ "VM-entry with events blocked by MOV SS.",
551	/* 27 */ "(Not Used)",
552	/* 28 */ "Invalid operand to INVEPT/INVVPID."
553	};
554	#endif /* VBOX_STRICT */
555
556
557
558	/**
559	* Updates the VM's last error record.
560	*
561	* If there was a VMX instruction error, reads the error data from the VMCS and
562	* updates VCPU's last error record as well.
563	*
564	* @param pVM The cross context VM structure.
565	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
566	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567	* VERR_VMX_INVALID_VMCS_FIELD.
568	* @param rc The error code.
569	*/
570	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571	{
572	AssertPtr(pVM);
573	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
575	{
576	AssertPtrReturnVoid(pVCpu);
577	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578	}
579	pVM->hm.s.lLastError = rc;
580	}
581
582
583	/**
584	* Reads the VM-entry interruption-information field from the VMCS into the VMX
585	* transient structure.
586	*
587	* @returns VBox status code.
588	* @param pVmxTransient Pointer to the VMX transient structure.
589	*
590	* @remarks No-long-jump zone!!!
591	*/
592	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593	{
594	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595	AssertRCReturn(rc, rc);
596	return VINF_SUCCESS;
597	}
598
599
600	/**
601	* Reads the VM-entry exception error code field from the VMCS into
602	* the VMX transient structure.
603	*
604	* @returns VBox status code.
605	* @param pVmxTransient Pointer to the VMX transient structure.
606	*
607	* @remarks No-long-jump zone!!!
608	*/
609	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612	AssertRCReturn(rc, rc);
613	return VINF_SUCCESS;
614	}
615
616
617	/**
618	* Reads the VM-entry exception error code field from the VMCS into
619	* the VMX transient structure.
620	*
621	* @returns VBox status code.
622	* @param pVmxTransient Pointer to the VMX transient structure.
623	*
624	* @remarks No-long-jump zone!!!
625	*/
626	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627	{
628	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629	AssertRCReturn(rc, rc);
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit interruption-information field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVmxTransient Pointer to the VMX transient structure.
640	*/
641	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642	{
643	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644	{
645	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646	AssertRCReturn(rc, rc);
647	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648	}
649	return VINF_SUCCESS;
650	}
651
652
653	/**
654	* Reads the VM-exit interruption error code from the VMCS into the VMX
655	* transient structure.
656	*
657	* @returns VBox status code.
658	* @param pVmxTransient Pointer to the VMX transient structure.
659	*/
660	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661	{
662	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665	AssertRCReturn(rc, rc);
666	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667	}
668	return VINF_SUCCESS;
669	}
670
671
672	/**
673	* Reads the VM-exit instruction length field from the VMCS into the VMX
674	* transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVmxTransient Pointer to the VMX transient structure.
678	*/
679	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680	{
681	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682	{
683	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684	AssertRCReturn(rc, rc);
685	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686	}
687	return VINF_SUCCESS;
688	}
689
690
691	/**
692	* Reads the VM-exit instruction-information field from the VMCS into
693	* the VMX transient structure.
694	*
695	* @returns VBox status code.
696	* @param pVmxTransient Pointer to the VMX transient structure.
697	*/
698	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699	{
700	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701	{
702	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703	AssertRCReturn(rc, rc);
704	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705	}
706	return VINF_SUCCESS;
707	}
708
709
710	/**
711	* Reads the exit code qualification from the VMCS into the VMX transient
712	* structure.
713	*
714	* @returns VBox status code.
715	* @param pVCpu The cross context virtual CPU structure of the
716	* calling EMT. (Required for the VMCS cache case.)
717	* @param pVmxTransient Pointer to the VMX transient structure.
718	*/
719	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720	{
721	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722	{
723	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724	AssertRCReturn(rc, rc);
725	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726	}
727	return VINF_SUCCESS;
728	}
729
730
731	/**
732	* Reads the IDT-vectoring information field from the VMCS into the VMX
733	* transient structure.
734	*
735	* @returns VBox status code.
736	* @param pVmxTransient Pointer to the VMX transient structure.
737	*
738	* @remarks No-long-jump zone!!!
739	*/
740	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741	{
742	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743	{
744	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745	AssertRCReturn(rc, rc);
746	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747	}
748	return VINF_SUCCESS;
749	}
750
751
752	/**
753	* Reads the IDT-vectoring error code from the VMCS into the VMX
754	* transient structure.
755	*
756	* @returns VBox status code.
757	* @param pVmxTransient Pointer to the VMX transient structure.
758	*/
759	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760	{
761	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762	{
763	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764	AssertRCReturn(rc, rc);
765	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766	}
767	return VINF_SUCCESS;
768	}
769
770
771	/**
772	* Enters VMX root mode operation on the current CPU.
773	*
774	* @returns VBox status code.
775	* @param pVM The cross context VM structure. Can be
776	* NULL, after a resume.
777	* @param HCPhysCpuPage Physical address of the VMXON region.
778	* @param pvCpuPage Pointer to the VMXON region.
779	*/
780	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781	{
782	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784	Assert(pvCpuPage);
785	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787	if (pVM)
788	{
789	/* Write the VMCS revision dword to the VMXON region. */
790	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791	}
792
793	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794	RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796	/* Enable the VMX bit in CR4 if necessary. */
797	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799	/* Enter VMX root mode. */
800	int rc = VMXEnable(HCPhysCpuPage);
801	if (RT_FAILURE(rc))
802	{
803	if (!(uOldCr4 & X86_CR4_VMXE))
804	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806	if (pVM)
807	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808	}
809
810	/* Restore interrupts. */
811	ASMSetFlags(fEFlags);
812	return rc;
813	}
814
815
816	/**
817	* Exits VMX root mode operation on the current CPU.
818	*
819	* @returns VBox status code.
820	*/
821	static int hmR0VmxLeaveRootMode(void)
822	{
823	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826	RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828	/* If we're for some reason not in VMX root mode, then don't leave it. */
829	RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831	int rc;
832	if (uHostCR4 & X86_CR4_VMXE)
833	{
834	/* Exit VMX root mode and clear the VMX bit in CR4. */
835	VMXDisable();
836	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837	rc = VINF_SUCCESS;
838	}
839	else
840	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842	/* Restore interrupts. */
843	ASMSetFlags(fEFlags);
844	return rc;
845	}
846
847
848	/**
849	* Allocates and maps one physically contiguous page. The allocated page is
850	* zero'd out. (Used by various VT-x structures).
851	*
852	* @returns IPRT status code.
853	* @param pMemObj Pointer to the ring-0 memory object.
854	* @param ppVirt Where to store the virtual address of the
855	* allocation.
856	* @param pHCPhys Where to store the physical address of the
857	* allocation.
858	*/
859	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860	{
861	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866	if (RT_FAILURE(rc))
867	return rc;
868	ppVirt = RTR0MemObjAddress(pMemObj);
869	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
870	ASMMemZero32(*ppVirt, PAGE_SIZE);
871	return VINF_SUCCESS;
872	}
873
874
875	/**
876	* Frees and unmaps an allocated physical page.
877	*
878	* @param pMemObj Pointer to the ring-0 memory object.
879	* @param ppVirt Where to re-initialize the virtual address of
880	* allocation as 0.
881	* @param pHCPhys Where to re-initialize the physical address of the
882	* allocation as 0.
883	*/
884	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885	{
886	AssertPtr(pMemObj);
887	AssertPtr(ppVirt);
888	AssertPtr(pHCPhys);
889	if (*pMemObj != NIL_RTR0MEMOBJ)
890	{
891	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
892	AssertRC(rc);
893	*pMemObj = NIL_RTR0MEMOBJ;
894	*ppVirt = 0;
895	*pHCPhys = 0;
896	}
897	}
898
899
900	/**
901	* Worker function to free VT-x related structures.
902	*
903	* @returns IPRT status code.
904	* @param pVM The cross context VM structure.
905	*/
906	static void hmR0VmxStructsFree(PVM pVM)
907	{
908	for (VMCPUID i = 0; i < pVM->cCpus; i++)
909	{
910	PVMCPU pVCpu = &pVM->aCpus[i];
911	AssertPtr(pVCpu);
912
913	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921	}
922
923	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926	#endif
927	}
928
929
930	/**
931	* Worker function to allocate VT-x related VM structures.
932	*
933	* @returns IPRT status code.
934	* @param pVM The cross context VM structure.
935	*/
936	static int hmR0VmxStructsAlloc(PVM pVM)
937	{
938	/*
939	* Initialize members up-front so we can cleanup properly on allocation failure.
940	*/
941	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944	pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953	#endif
954	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957	for (VMCPUID i = 0; i < pVM->cCpus; i++)
958	{
959	PVMCPU pVCpu = &pVM->aCpus[i];
960	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965	}
966	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967	#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974	/*
975	* Allocate all the VT-x structures.
976	*/
977	int rc = VINF_SUCCESS;
978	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980	if (RT_FAILURE(rc))
981	goto cleanup;
982	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984	#endif
985
986	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988	{
989	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990	&pVM->hm.s.vmx.HCPhysApicAccess);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	/*
996	* Initialize per-VCPU VT-x structures.
997	*/
998	for (VMCPUID i = 0; i < pVM->cCpus; i++)
999	{
1000	PVMCPU pVCpu = &pVM->aCpus[i];
1001	AssertPtr(pVCpu);
1002
1003	/* Allocate the VM control structure (VMCS). */
1004	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005	if (RT_FAILURE(rc))
1006	goto cleanup;
1007
1008	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010	{
1011	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012	&pVCpu->hm.s.vmx.HCPhysVirtApic);
1013	if (RT_FAILURE(rc))
1014	goto cleanup;
1015	}
1016
1017	/*
1018	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019	* transparent accesses of specific MSRs.
1020	*
1021	* If the condition for enabling MSR bitmaps changes here, don't forget to
1022	* update HMAreMsrBitmapsAvailable().
1023	*/
1024	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025	{
1026	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028	if (RT_FAILURE(rc))
1029	goto cleanup;
1030	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031	}
1032
1033	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035	if (RT_FAILURE(rc))
1036	goto cleanup;
1037
1038	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1039	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040	if (RT_FAILURE(rc))
1041	goto cleanup;
1042	}
1043
1044	return VINF_SUCCESS;
1045
1046	cleanup:
1047	hmR0VmxStructsFree(pVM);
1048	return rc;
1049	}
1050
1051
1052	/**
1053	* Does global VT-x initialization (called during module initialization).
1054	*
1055	* @returns VBox status code.
1056	*/
1057	VMMR0DECL(int) VMXR0GlobalInit(void)
1058	{
1059	#ifdef HMVMX_USE_FUNCTION_TABLE
1060	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061	# ifdef VBOX_STRICT
1062	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063	Assert(g_apfnVMExitHandlers[i]);
1064	# endif
1065	#endif
1066	return VINF_SUCCESS;
1067	}
1068
1069
1070	/**
1071	* Does global VT-x termination (called during module termination).
1072	*/
1073	VMMR0DECL(void) VMXR0GlobalTerm()
1074	{
1075	/* Nothing to do currently. */
1076	}
1077
1078
1079	/**
1080	* Sets up and activates VT-x on the current CPU.
1081	*
1082	* @returns VBox status code.
1083	* @param pCpu Pointer to the global CPU info struct.
1084	* @param pVM The cross context VM structure. Can be
1085	* NULL after a host resume operation.
1086	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087	* fEnabledByHost is @c true).
1088	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089	* @a fEnabledByHost is @c true).
1090	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091	* enable VT-x on the host.
1092	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1093	*/
1094	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095	void *pvMsrs)
1096	{
1097	Assert(pCpu);
1098	Assert(pvMsrs);
1099	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101	/* Enable VT-x if it's not already enabled by the host. */
1102	if (!fEnabledByHost)
1103	{
1104	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105	if (RT_FAILURE(rc))
1106	return rc;
1107	}
1108
1109	/*
1110	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112	*/
1113	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115	{
1116	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117	pCpu->fFlushAsidBeforeUse = false;
1118	}
1119	else
1120	pCpu->fFlushAsidBeforeUse = true;
1121
1122	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123	++pCpu->cTlbFlushes;
1124
1125	return VINF_SUCCESS;
1126	}
1127
1128
1129	/**
1130	* Deactivates VT-x on the current CPU.
1131	*
1132	* @returns VBox status code.
1133	* @param pCpu Pointer to the global CPU info struct.
1134	* @param pvCpuPage Pointer to the VMXON region.
1135	* @param HCPhysCpuPage Physical address of the VMXON region.
1136	*
1137	* @remarks This function should never be called when SUPR0EnableVTx() or
1138	* similar was used to enable VT-x on the host.
1139	*/
1140	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141	{
1142	NOREF(pCpu);
1143	NOREF(pvCpuPage);
1144	NOREF(HCPhysCpuPage);
1145
1146	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147	return hmR0VmxLeaveRootMode();
1148	}
1149
1150
1151	/**
1152	* Sets the permission bits for the specified MSR in the MSR bitmap.
1153	*
1154	* @param pVCpu The cross context virtual CPU structure.
1155	* @param uMsr The MSR value.
1156	* @param enmRead Whether reading this MSR causes a VM-exit.
1157	* @param enmWrite Whether writing this MSR causes a VM-exit.
1158	*/
1159	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160	{
1161	int32_t iBit;
1162	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164	/*
1165	* Layout:
1166	* 0x000 - 0x3ff - Low MSR read bits
1167	* 0x400 - 0x7ff - High MSR read bits
1168	* 0x800 - 0xbff - Low MSR write bits
1169	* 0xc00 - 0xfff - High MSR write bits
1170	*/
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1181
1182	Assert(iBit <= 0x1fff);
1183	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1184	ASMBitSet(pbMsrBitmap, iBit);
1185	else
1186	ASMBitClear(pbMsrBitmap, iBit);
1187
1188	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1189	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1190	else
1191	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1192	}
1193
1194
1195	#ifdef VBOX_STRICT
1196	/**
1197	* Gets the permission bits for the specified MSR in the MSR bitmap.
1198	*
1199	* @returns VBox status code.
1200	* @retval VINF_SUCCESS if the specified MSR is found.
1201	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1202	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1203	*
1204	* @param pVCpu The cross context virtual CPU structure.
1205	* @param uMsr The MSR.
1206	* @param penmRead Where to store the read permissions.
1207	* @param penmWrite Where to store the write permissions.
1208	*/
1209	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1210	{
1211	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1212	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1213	int32_t iBit;
1214	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1215
1216	/* See hmR0VmxSetMsrPermission() for the layout. */
1217	if (uMsr <= 0x00001FFF)
1218	iBit = uMsr;
1219	else if ( uMsr >= 0xC0000000
1220	&& uMsr <= 0xC0001FFF)
1221	{
1222	iBit = (uMsr - 0xC0000000);
1223	pbMsrBitmap += 0x400;
1224	}
1225	else
1226	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1227
1228	Assert(iBit <= 0x1fff);
1229	if (ASMBitTest(pbMsrBitmap, iBit))
1230	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1231	else
1232	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1233
1234	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1235	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1236	else
1237	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1238	return VINF_SUCCESS;
1239	}
1240	#endif /* VBOX_STRICT */
1241
1242
1243	/**
1244	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1245	* area.
1246	*
1247	* @returns VBox status code.
1248	* @param pVCpu The cross context virtual CPU structure.
1249	* @param cMsrs The number of MSRs.
1250	*/
1251	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1252	{
1253	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1254	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1255	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1256	{
1257	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1258	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1259	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1260	}
1261
1262	/* Update number of guest MSRs to load/store across the world-switch. */
1263	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1264	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1265
1266	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1267	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1268
1269	/* Update the VCPU's copy of the MSR count. */
1270	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272	return VINF_SUCCESS;
1273	}
1274
1275
1276	/**
1277	* Adds a new (or updates the value of an existing) guest/host MSR
1278	* pair to be swapped during the world-switch as part of the
1279	* auto-load/store MSR area in the VMCS.
1280	*
1281	* @returns VBox status code.
1282	* @param pVCpu The cross context virtual CPU structure.
1283	* @param uMsr The MSR.
1284	* @param uGuestMsrValue Value of the guest MSR.
1285	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1286	* necessary.
1287	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288	* its value was updated. Optional, can be NULL.
1289	*/
1290	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291	bool *pfAddedAndUpdated)
1292	{
1293	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295	uint32_t i;
1296	for (i = 0; i < cMsrs; i++)
1297	{
1298	if (pGuestMsr->u32Msr == uMsr)
1299	break;
1300	pGuestMsr++;
1301	}
1302
1303	bool fAdded = false;
1304	if (i == cMsrs)
1305	{
1306	++cMsrs;
1307	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314	fAdded = true;
1315	}
1316
1317	/* Update the MSR values in the auto-load/store MSR area. */
1318	pGuestMsr->u32Msr = uMsr;
1319	pGuestMsr->u64Value = uGuestMsrValue;
1320
1321	/* Create/update the MSR slot in the host MSR area. */
1322	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323	pHostMsr += i;
1324	pHostMsr->u32Msr = uMsr;
1325
1326	/*
1327	* Update the host MSR only when requested by the caller AND when we're
1328	* adding it to the auto-load/store area. Otherwise, it would have been
1329	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330	*/
1331	bool fUpdatedMsrValue = false;
1332	if ( fAdded
1333	&& fUpdateHostMsr)
1334	{
1335	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338	fUpdatedMsrValue = true;
1339	}
1340
1341	if (pfAddedAndUpdated)
1342	*pfAddedAndUpdated = fUpdatedMsrValue;
1343	return VINF_SUCCESS;
1344	}
1345
1346
1347	/**
1348	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1349	* auto-load/store MSR area in the VMCS.
1350	*
1351	* @returns VBox status code.
1352	* @param pVCpu The cross context virtual CPU structure.
1353	* @param uMsr The MSR.
1354	*/
1355	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356	{
1357	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359	for (uint32_t i = 0; i < cMsrs; i++)
1360	{
1361	/* Find the MSR. */
1362	if (pGuestMsr->u32Msr == uMsr)
1363	{
1364	/* If it's the last MSR, simply reduce the count. */
1365	if (i == cMsrs - 1)
1366	{
1367	--cMsrs;
1368	break;
1369	}
1370
1371	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373	pLastGuestMsr += cMsrs - 1;
1374	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379	pLastHostMsr += cMsrs - 1;
1380	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381	pHostMsr->u64Value = pLastHostMsr->u64Value;
1382	--cMsrs;
1383	break;
1384	}
1385	pGuestMsr++;
1386	}
1387
1388	/* Update the VMCS if the count changed (meaning the MSR was found). */
1389	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390	{
1391	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392	AssertRCReturn(rc, rc);
1393
1394	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399	return VINF_SUCCESS;
1400	}
1401
1402	return VERR_NOT_FOUND;
1403	}
1404
1405
1406	/**
1407	* Checks if the specified guest MSR is part of the auto-load/store area in
1408	* the VMCS.
1409	*
1410	* @returns true if found, false otherwise.
1411	* @param pVCpu The cross context virtual CPU structure.
1412	* @param uMsr The MSR to find.
1413	*/
1414	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415	{
1416	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420	{
1421	if (pGuestMsr->u32Msr == uMsr)
1422	return true;
1423	}
1424	return false;
1425	}
1426
1427
1428	/**
1429	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430	*
1431	* @param pVCpu The cross context virtual CPU structure.
1432	*
1433	* @remarks No-long-jump zone!!!
1434	*/
1435	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436	{
1437	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443	{
1444	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446	/*
1447	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449	*/
1450	if (pHostMsr->u32Msr == MSR_K6_EFER)
1451	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452	else
1453	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454	}
1455
1456	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457	}
1458
1459
1460	#if HC_ARCH_BITS == 64
1461	/**
1462	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1463	* perform lazy restoration of the host MSRs while leaving VT-x.
1464	*
1465	* @param pVCpu The cross context virtual CPU structure.
1466	*
1467	* @remarks No-long-jump zone!!!
1468	*/
1469	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1470	{
1471	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1472
1473	/*
1474	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1475	*/
1476	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1477	{
1478	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1479	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1480	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1481	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1482	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1483	}
1484	}
1485
1486
1487	/**
1488	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1489	* lazily while leaving VT-x.
1490	*
1491	* @returns true if it does, false otherwise.
1492	* @param pVCpu The cross context virtual CPU structure.
1493	* @param uMsr The MSR to check.
1494	*/
1495	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1496	{
1497	NOREF(pVCpu);
1498	switch (uMsr)
1499	{
1500	case MSR_K8_LSTAR:
1501	case MSR_K6_STAR:
1502	case MSR_K8_SF_MASK:
1503	case MSR_K8_KERNEL_GS_BASE:
1504	return true;
1505	}
1506	return false;
1507	}
1508
1509
1510	/**
1511	* Saves a set of guest MSRs back into the guest-CPU context.
1512	*
1513	* @param pVCpu The cross context virtual CPU structure.
1514	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1515	* out-of-sync. Make sure to update the required fields
1516	* before using them.
1517	*
1518	* @remarks No-long-jump zone!!!
1519	*/
1520	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1521	{
1522	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1523	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1524
1525	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1526	{
1527	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1528	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1529	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1530	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1531	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1532	}
1533	}
1534
1535
1536	/**
1537	* Loads a set of guests MSRs to allow read/passthru to the guest.
1538	*
1539	* The name of this function is slightly confusing. This function does NOT
1540	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1541	* common prefix for functions dealing with "lazy restoration" of the shared
1542	* MSRs.
1543	*
1544	* @param pVCpu The cross context virtual CPU structure.
1545	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1546	* out-of-sync. Make sure to update the required fields
1547	* before using them.
1548	*
1549	* @remarks No-long-jump zone!!!
1550	*/
1551	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1552	{
1553	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1554	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1555
1556	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1557	do { \
1558	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1559	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1560	else \
1561	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1562	} while (0)
1563
1564	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1565	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1566	{
1567	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1568	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1569	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1570	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1571	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1572	}
1573	else
1574	{
1575	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1576	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1577	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1578	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1579	}
1580
1581	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1582	}
1583
1584
1585	/**
1586	* Performs lazy restoration of the set of host MSRs if they were previously
1587	* loaded with guest MSR values.
1588	*
1589	* @param pVCpu The cross context virtual CPU structure.
1590	*
1591	* @remarks No-long-jump zone!!!
1592	* @remarks The guest MSRs should have been saved back into the guest-CPU
1593	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1594	*/
1595	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1596	{
1597	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1598	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1599
1600	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1601	{
1602	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1603	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1604	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1605	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1606	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1607	}
1608	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1609	}
1610	#endif /* HC_ARCH_BITS == 64 */
1611
1612
1613	/**
1614	* Verifies that our cached values of the VMCS controls are all
1615	* consistent with what's actually present in the VMCS.
1616	*
1617	* @returns VBox status code.
1618	* @param pVCpu The cross context virtual CPU structure.
1619	*/
1620	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1621	{
1622	uint32_t u32Val;
1623	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1624	AssertRCReturn(rc, rc);
1625	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1626	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1627
1628	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1629	AssertRCReturn(rc, rc);
1630	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1631	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1632
1633	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1634	AssertRCReturn(rc, rc);
1635	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1636	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1637
1638	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1639	AssertRCReturn(rc, rc);
1640	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1641	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1642
1643	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1644	{
1645	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1646	AssertRCReturn(rc, rc);
1647	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1648	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1649	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1650	}
1651
1652	return VINF_SUCCESS;
1653	}
1654
1655
1656	#ifdef VBOX_STRICT
1657	/**
1658	* Verifies that our cached host EFER value has not changed
1659	* since we cached it.
1660	*
1661	* @param pVCpu The cross context virtual CPU structure.
1662	*/
1663	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1664	{
1665	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1666
1667	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1668	{
1669	uint64_t u64Val;
1670	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1671	AssertRC(rc);
1672
1673	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1674	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1675	}
1676	}
1677
1678
1679	/**
1680	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1681	* VMCS are correct.
1682	*
1683	* @param pVCpu The cross context virtual CPU structure.
1684	*/
1685	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1686	{
1687	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1688
1689	/* Verify MSR counts in the VMCS are what we think it should be. */
1690	uint32_t cMsrs;
1691	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1692	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1693
1694	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1695	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1696
1697	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1698	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1699
1700	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1701	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1702	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1703	{
1704	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1705	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1706	pGuestMsr->u32Msr, cMsrs));
1707
1708	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1709	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1710	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1711
1712	/* Verify that the permissions are as expected in the MSR bitmap. */
1713	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1714	{
1715	VMXMSREXITREAD enmRead;
1716	VMXMSREXITWRITE enmWrite;
1717	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1718	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1719	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1720	{
1721	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1722	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1723	}
1724	else
1725	{
1726	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1727	pGuestMsr->u32Msr, cMsrs));
1728	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1729	pGuestMsr->u32Msr, cMsrs));
1730	}
1731	}
1732	}
1733	}
1734	#endif /* VBOX_STRICT */
1735
1736
1737	/**
1738	* Flushes the TLB using EPT.
1739	*
1740	* @returns VBox status code.
1741	* @param pVCpu The cross context virtual CPU structure of the calling
1742	* EMT. Can be NULL depending on @a enmFlush.
1743	* @param enmFlush Type of flush.
1744	*
1745	* @remarks Caller is responsible for making sure this function is called only
1746	* when NestedPaging is supported and providing @a enmFlush that is
1747	* supported by the CPU.
1748	* @remarks Can be called with interrupts disabled.
1749	*/
1750	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1751	{
1752	uint64_t au64Descriptor[2];
1753	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1754	au64Descriptor[0] = 0;
1755	else
1756	{
1757	Assert(pVCpu);
1758	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1759	}
1760	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1761
1762	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1763	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1764	rc));
1765	if ( RT_SUCCESS(rc)
1766	&& pVCpu)
1767	{
1768	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1769	}
1770	}
1771
1772
1773	/**
1774	* Flushes the TLB using VPID.
1775	*
1776	* @returns VBox status code.
1777	* @param pVM The cross context VM structure.
1778	* @param pVCpu The cross context virtual CPU structure of the calling
1779	* EMT. Can be NULL depending on @a enmFlush.
1780	* @param enmFlush Type of flush.
1781	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1782	* on @a enmFlush).
1783	*
1784	* @remarks Can be called with interrupts disabled.
1785	*/
1786	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1787	{
1788	NOREF(pVM);
1789	AssertPtr(pVM);
1790	Assert(pVM->hm.s.vmx.fVpid);
1791
1792	uint64_t au64Descriptor[2];
1793	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1794	{
1795	au64Descriptor[0] = 0;
1796	au64Descriptor[1] = 0;
1797	}
1798	else
1799	{
1800	AssertPtr(pVCpu);
1801	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1802	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1803	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1804	au64Descriptor[1] = GCPtr;
1805	}
1806
1807	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1808	AssertMsg(rc == VINF_SUCCESS,
1809	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1810	if ( RT_SUCCESS(rc)
1811	&& pVCpu)
1812	{
1813	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1814	}
1815	}
1816
1817
1818	/**
1819	* Invalidates a guest page by guest virtual address. Only relevant for
1820	* EPT/VPID, otherwise there is nothing really to invalidate.
1821	*
1822	* @returns VBox status code.
1823	* @param pVM The cross context VM structure.
1824	* @param pVCpu The cross context virtual CPU structure.
1825	* @param GCVirt Guest virtual address of the page to invalidate.
1826	*/
1827	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1828	{
1829	AssertPtr(pVM);
1830	AssertPtr(pVCpu);
1831	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1832
1833	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1834	if (!fFlushPending)
1835	{
1836	/*
1837	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1838	* See @bugref{6043} and @bugref{6177}.
1839	*
1840	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1841	* function maybe called in a loop with individual addresses.
1842	*/
1843	if (pVM->hm.s.vmx.fVpid)
1844	{
1845	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1846	{
1847	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1848	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1849	}
1850	else
1851	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1852	}
1853	else if (pVM->hm.s.fNestedPaging)
1854	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1855	}
1856
1857	return VINF_SUCCESS;
1858	}
1859
1860
1861	/**
1862	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1863	* otherwise there is nothing really to invalidate.
1864	*
1865	* @returns VBox status code.
1866	* @param pVM The cross context VM structure.
1867	* @param pVCpu The cross context virtual CPU structure.
1868	* @param GCPhys Guest physical address of the page to invalidate.
1869	*/
1870	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1871	{
1872	NOREF(pVM); NOREF(GCPhys);
1873	LogFlowFunc(("%RGp\n", GCPhys));
1874
1875	/*
1876	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1877	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1878	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1879	*/
1880	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1881	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1882	return VINF_SUCCESS;
1883	}
1884
1885
1886	/**
1887	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1888	* case where neither EPT nor VPID is supported by the CPU.
1889	*
1890	* @param pVM The cross context VM structure.
1891	* @param pVCpu The cross context virtual CPU structure.
1892	* @param pCpu Pointer to the global HM struct.
1893	*
1894	* @remarks Called with interrupts disabled.
1895	*/
1896	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1897	{
1898	AssertPtr(pVCpu);
1899	AssertPtr(pCpu);
1900	NOREF(pVM);
1901
1902	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1903
1904	Assert(pCpu->idCpu != NIL_RTCPUID);
1905	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1906	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1907	pVCpu->hm.s.fForceTLBFlush = false;
1908	return;
1909	}
1910
1911
1912	/**
1913	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1914	*
1915	* @param pVM The cross context VM structure.
1916	* @param pVCpu The cross context virtual CPU structure.
1917	* @param pCpu Pointer to the global HM CPU struct.
1918	* @remarks All references to "ASID" in this function pertains to "VPID" in
1919	* Intel's nomenclature. The reason is, to avoid confusion in compare
1920	* statements since the host-CPU copies are named "ASID".
1921	*
1922	* @remarks Called with interrupts disabled.
1923	*/
1924	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1925	{
1926	#ifdef VBOX_WITH_STATISTICS
1927	bool fTlbFlushed = false;
1928	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1929	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1930	if (!fTlbFlushed) \
1931	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1932	} while (0)
1933	#else
1934	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1935	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1936	#endif
1937
1938	AssertPtr(pVM);
1939	AssertPtr(pCpu);
1940	AssertPtr(pVCpu);
1941	Assert(pCpu->idCpu != NIL_RTCPUID);
1942
1943	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1944	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1945	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1946
1947	/*
1948	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1949	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1950	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1951	*/
1952	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1953	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1954	{
1955	++pCpu->uCurrentAsid;
1956	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1957	{
1958	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1959	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1960	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1961	}
1962
1963	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1964	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1965	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1966
1967	/*
1968	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1969	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1970	*/
1971	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1972	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1973	HMVMX_SET_TAGGED_TLB_FLUSHED();
1974	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1975	}
1976
1977	/* Check for explicit TLB flushes. */
1978	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1979	{
1980	/*
1981	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1982	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1983	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1984	* but not guest-physical mappings.
1985	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1986	*/
1987	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1988	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1989	HMVMX_SET_TAGGED_TLB_FLUSHED();
1990	}
1991
1992	pVCpu->hm.s.fForceTLBFlush = false;
1993	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1994
1995	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1996	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1997	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1998	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1999	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2000	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2001	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2002	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2003	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2004
2005	/* Update VMCS with the VPID. */
2006	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2007	AssertRC(rc);
2008
2009	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2010	}
2011
2012
2013	/**
2014	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2015	*
2016	* @returns VBox status code.
2017	* @param pVM The cross context VM structure.
2018	* @param pVCpu The cross context virtual CPU structure.
2019	* @param pCpu Pointer to the global HM CPU struct.
2020	*
2021	* @remarks Called with interrupts disabled.
2022	*/
2023	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2024	{
2025	AssertPtr(pVM);
2026	AssertPtr(pVCpu);
2027	AssertPtr(pCpu);
2028	Assert(pCpu->idCpu != NIL_RTCPUID);
2029	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2030	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2031
2032	/*
2033	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2034	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2035	*/
2036	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2037	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2038	{
2039	pVCpu->hm.s.fForceTLBFlush = true;
2040	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2041	}
2042
2043	/* Check for explicit TLB flushes. */
2044	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2045	{
2046	pVCpu->hm.s.fForceTLBFlush = true;
2047	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2048	}
2049
2050	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2051	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2052
2053	if (pVCpu->hm.s.fForceTLBFlush)
2054	{
2055	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2056	pVCpu->hm.s.fForceTLBFlush = false;
2057	}
2058	}
2059
2060
2061	/**
2062	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2063	*
2064	* @returns VBox status code.
2065	* @param pVM The cross context VM structure.
2066	* @param pVCpu The cross context virtual CPU structure.
2067	* @param pCpu Pointer to the global HM CPU struct.
2068	*
2069	* @remarks Called with interrupts disabled.
2070	*/
2071	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2072	{
2073	AssertPtr(pVM);
2074	AssertPtr(pVCpu);
2075	AssertPtr(pCpu);
2076	Assert(pCpu->idCpu != NIL_RTCPUID);
2077	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2078	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2079
2080	/*
2081	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2082	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2083	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2084	*/
2085	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2086	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2087	{
2088	pVCpu->hm.s.fForceTLBFlush = true;
2089	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2090	}
2091
2092	/* Check for explicit TLB flushes. */
2093	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2094	{
2095	/*
2096	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2097	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2098	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2099	*/
2100	pVCpu->hm.s.fForceTLBFlush = true;
2101	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2102	}
2103
2104	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2105	if (pVCpu->hm.s.fForceTLBFlush)
2106	{
2107	++pCpu->uCurrentAsid;
2108	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2109	{
2110	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2111	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2112	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2113	}
2114
2115	pVCpu->hm.s.fForceTLBFlush = false;
2116	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2117	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2118	if (pCpu->fFlushAsidBeforeUse)
2119	{
2120	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2121	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2122	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2123	{
2124	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2125	pCpu->fFlushAsidBeforeUse = false;
2126	}
2127	else
2128	{
2129	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2130	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2131	}
2132	}
2133	}
2134
2135	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2136	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2137	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2138	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2139	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2140	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2141	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2142
2143	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2144	AssertRC(rc);
2145	}
2146
2147
2148	/**
2149	* Flushes the guest TLB entry based on CPU capabilities.
2150	*
2151	* @param pVCpu The cross context virtual CPU structure.
2152	* @param pCpu Pointer to the global HM CPU struct.
2153	*/
2154	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2155	{
2156	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2157	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2158	#endif
2159	PVM pVM = pVCpu->CTX_SUFF(pVM);
2160	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2161	{
2162	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2163	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2164	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2165	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2166	default:
2167	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2168	break;
2169	}
2170
2171	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2172	}
2173
2174
2175	/**
2176	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2177	* TLB entries from the host TLB before VM-entry.
2178	*
2179	* @returns VBox status code.
2180	* @param pVM The cross context VM structure.
2181	*/
2182	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2183	{
2184	/*
2185	* Determine optimal flush type for Nested Paging.
2186	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2187	* guest execution (see hmR3InitFinalizeR0()).
2188	*/
2189	if (pVM->hm.s.fNestedPaging)
2190	{
2191	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2192	{
2193	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2194	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2195	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2196	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2197	else
2198	{
2199	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2200	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2201	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2202	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2203	}
2204
2205	/* Make sure the write-back cacheable memory type for EPT is supported. */
2206	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2207	{
2208	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2209	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2210	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2211	}
2212
2213	/* EPT requires a page-walk length of 4. */
2214	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2215	{
2216	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2217	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2218	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2219	}
2220	}
2221	else
2222	{
2223	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2224	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2225	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2226	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2227	}
2228	}
2229
2230	/*
2231	* Determine optimal flush type for VPID.
2232	*/
2233	if (pVM->hm.s.vmx.fVpid)
2234	{
2235	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2236	{
2237	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2238	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2239	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2240	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2241	else
2242	{
2243	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2244	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2245	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2246	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2247	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2248	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2249	pVM->hm.s.vmx.fVpid = false;
2250	}
2251	}
2252	else
2253	{
2254	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2255	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2256	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2257	pVM->hm.s.vmx.fVpid = false;
2258	}
2259	}
2260
2261	/*
2262	* Setup the handler for flushing tagged-TLBs.
2263	*/
2264	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2265	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2266	else if (pVM->hm.s.fNestedPaging)
2267	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2268	else if (pVM->hm.s.vmx.fVpid)
2269	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2270	else
2271	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2272	return VINF_SUCCESS;
2273	}
2274
2275
2276	/**
2277	* Sets up pin-based VM-execution controls in the VMCS.
2278	*
2279	* @returns VBox status code.
2280	* @param pVM The cross context VM structure.
2281	* @param pVCpu The cross context virtual CPU structure.
2282	*/
2283	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2284	{
2285	AssertPtr(pVM);
2286	AssertPtr(pVCpu);
2287
2288	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2289	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2290
2291	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2292	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2293
2294	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2295	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2296
2297	/* Enable the VMX preemption timer. */
2298	if (pVM->hm.s.vmx.fUsePreemptTimer)
2299	{
2300	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2301	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2302	}
2303
2304	if ((val & zap) != val)
2305	{
2306	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2307	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2308	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2309	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2310	}
2311
2312	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2313	AssertRCReturn(rc, rc);
2314
2315	pVCpu->hm.s.vmx.u32PinCtls = val;
2316	return rc;
2317	}
2318
2319
2320	/**
2321	* Sets up processor-based VM-execution controls in the VMCS.
2322	*
2323	* @returns VBox status code.
2324	* @param pVM The cross context VM structure.
2325	* @param pVCpu The cross context virtual CPU structure.
2326	*/
2327	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2328	{
2329	AssertPtr(pVM);
2330	AssertPtr(pVCpu);
2331
2332	int rc = VERR_INTERNAL_ERROR_5;
2333	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2334	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2335
2336	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2337	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2338	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2339	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2340	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2341	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2342	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2343
2344	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2345	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2346	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2347	{
2348	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2349	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2350	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2351	}
2352
2353	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2354	if (!pVM->hm.s.fNestedPaging)
2355	{
2356	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2357	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2358	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2359	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2360	}
2361
2362	/* Use TPR shadowing if supported by the CPU. */
2363	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2364	{
2365	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2366	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2367	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2368	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2369	AssertRCReturn(rc, rc);
2370
2371	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2372	/* CR8 writes cause a VM-exit based on TPR threshold. */
2373	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2374	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2375	}
2376	else
2377	{
2378	/*
2379	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2380	* Set this control only for 64-bit guests.
2381	*/
2382	if (pVM->hm.s.fAllow64BitGuests)
2383	{
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2385	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2386	}
2387	}
2388
2389	/* Use MSR-bitmaps if supported by the CPU. */
2390	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2391	{
2392	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2393
2394	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2395	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2396	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2397	AssertRCReturn(rc, rc);
2398
2399	/*
2400	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2401	* automatically using dedicated fields in the VMCS.
2402	*/
2403	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2404	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2405	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2406	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2407	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2408
2409	#if HC_ARCH_BITS == 64
2410	/*
2411	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2412	*/
2413	if (pVM->hm.s.fAllow64BitGuests)
2414	{
2415	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2416	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2417	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2418	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2419	}
2420	#endif
2421	}
2422
2423	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2424	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2425	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2426
2427	if ((val & zap) != val)
2428	{
2429	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2430	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2431	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2432	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2433	}
2434
2435	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2436	AssertRCReturn(rc, rc);
2437
2438	pVCpu->hm.s.vmx.u32ProcCtls = val;
2439
2440	/*
2441	* Secondary processor-based VM-execution controls.
2442	*/
2443	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2444	{
2445	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2446	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2447
2448	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2449	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2450
2451	if (pVM->hm.s.fNestedPaging)
2452	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2453	else
2454	{
2455	/*
2456	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2457	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2458	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2459	*/
2460	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2461	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2462	}
2463
2464	if (pVM->hm.s.vmx.fVpid)
2465	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2466
2467	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2468	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2469
2470	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2471	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2472	* done dynamically. */
2473	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2474	{
2475	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2476	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2477	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2478	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2479	AssertRCReturn(rc, rc);
2480	}
2481
2482	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2483	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2484
2485	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2486	&& pVM->hm.s.vmx.cPleGapTicks
2487	&& pVM->hm.s.vmx.cPleWindowTicks)
2488	{
2489	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2490
2491	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2492	AssertRCReturn(rc, rc);
2493
2494	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2495	AssertRCReturn(rc, rc);
2496	}
2497
2498	if ((val & zap) != val)
2499	{
2500	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2501	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2502	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2503	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2504	}
2505
2506	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2507	AssertRCReturn(rc, rc);
2508
2509	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2510	}
2511	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2512	{
2513	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2514	"available\n"));
2515	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2516	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2517	}
2518
2519	return VINF_SUCCESS;
2520	}
2521
2522
2523	/**
2524	* Sets up miscellaneous (everything other than Pin & Processor-based
2525	* VM-execution) control fields in the VMCS.
2526	*
2527	* @returns VBox status code.
2528	* @param pVM The cross context VM structure.
2529	* @param pVCpu The cross context virtual CPU structure.
2530	*/
2531	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2532	{
2533	NOREF(pVM);
2534	AssertPtr(pVM);
2535	AssertPtr(pVCpu);
2536
2537	int rc = VERR_GENERAL_FAILURE;
2538
2539	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2540	#if 0
2541	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2542	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2543	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2544
2545	/*
2546	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2547	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2548	* We thus use the exception bitmap to control it rather than use both.
2549	*/
2550	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2551	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2552
2553	/** @todo Explore possibility of using IO-bitmaps. */
2554	/* All IO & IOIO instructions cause VM-exits. */
2555	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2556	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2557
2558	/* Initialize the MSR-bitmap area. */
2559	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2560	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2561	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2562	#endif
2563
2564	/* Setup MSR auto-load/store area. */
2565	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2566	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2567	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2568	AssertRCReturn(rc, rc);
2569	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2570	AssertRCReturn(rc, rc);
2571
2572	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2573	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2574	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2575	AssertRCReturn(rc, rc);
2576
2577	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2578	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2579	AssertRCReturn(rc, rc);
2580
2581	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2582	#if 0
2583	/* Setup debug controls */
2584	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2585	AssertRCReturn(rc, rc);
2586	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2587	AssertRCReturn(rc, rc);
2588	#endif
2589
2590	return rc;
2591	}
2592
2593
2594	/**
2595	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2596	*
2597	* @returns VBox status code.
2598	* @param pVM The cross context VM structure.
2599	* @param pVCpu The cross context virtual CPU structure.
2600	*/
2601	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2602	{
2603	AssertPtr(pVM);
2604	AssertPtr(pVCpu);
2605
2606	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2607
2608	uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2609
2610	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2611	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2612
2613	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2614	and writes, and because recursive #DBs can cause the CPU hang, we must always
2615	intercept #DB. */
2616	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2617
2618	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2619	if (!pVM->hm.s.fNestedPaging)
2620	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2621
2622	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2623	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2624	AssertRCReturn(rc, rc);
2625	return rc;
2626	}
2627
2628
2629	/**
2630	* Sets up the initial guest-state mask. The guest-state mask is consulted
2631	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2632	* for the nested virtualization case (as it would cause a VM-exit).
2633	*
2634	* @param pVCpu The cross context virtual CPU structure.
2635	*/
2636	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2637	{
2638	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2639	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2640	return VINF_SUCCESS;
2641	}
2642
2643
2644	/**
2645	* Does per-VM VT-x initialization.
2646	*
2647	* @returns VBox status code.
2648	* @param pVM The cross context VM structure.
2649	*/
2650	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2651	{
2652	LogFlowFunc(("pVM=%p\n", pVM));
2653
2654	int rc = hmR0VmxStructsAlloc(pVM);
2655	if (RT_FAILURE(rc))
2656	{
2657	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2658	return rc;
2659	}
2660
2661	return VINF_SUCCESS;
2662	}
2663
2664
2665	/**
2666	* Does per-VM VT-x termination.
2667	*
2668	* @returns VBox status code.
2669	* @param pVM The cross context VM structure.
2670	*/
2671	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2672	{
2673	LogFlowFunc(("pVM=%p\n", pVM));
2674
2675	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2676	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2677	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2678	#endif
2679	hmR0VmxStructsFree(pVM);
2680	return VINF_SUCCESS;
2681	}
2682
2683
2684	/**
2685	* Sets up the VM for execution under VT-x.
2686	* This function is only called once per-VM during initialization.
2687	*
2688	* @returns VBox status code.
2689	* @param pVM The cross context VM structure.
2690	*/
2691	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2692	{
2693	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2694	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2695
2696	LogFlowFunc(("pVM=%p\n", pVM));
2697
2698	/*
2699	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2700	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2701	*/
2702	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2703	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2704	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2705	{
2706	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2707	return VERR_INTERNAL_ERROR;
2708	}
2709
2710	/* Initialize these always, see hmR3InitFinalizeR0().*/
2711	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2712	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2713
2714	/* Setup the tagged-TLB flush handlers. */
2715	int rc = hmR0VmxSetupTaggedTlb(pVM);
2716	if (RT_FAILURE(rc))
2717	{
2718	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2719	return rc;
2720	}
2721
2722	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2723	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2724	#if HC_ARCH_BITS == 64
2725	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2726	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2727	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2728	{
2729	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2730	}
2731	#endif
2732
2733	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2734	{
2735	PVMCPU pVCpu = &pVM->aCpus[i];
2736	AssertPtr(pVCpu);
2737	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2738
2739	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2740	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2741
2742	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2743	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2744	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2745
2746	/* Set revision dword at the beginning of the VMCS structure. */
2747	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2748
2749	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2750	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2751	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2752	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2753
2754	/* Load this VMCS as the current VMCS. */
2755	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2756	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2757	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2758
2759	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2760	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2761	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2762
2763	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2764	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2765	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2766
2767	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2768	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2769	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2770
2771	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2772	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2773	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2774
2775	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2776	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2777	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2778
2779	#if HC_ARCH_BITS == 32
2780	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2781	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2782	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2783	#endif
2784
2785	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2786	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2787	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2788	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2789
2790	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2791
2792	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2793	}
2794
2795	return VINF_SUCCESS;
2796	}
2797
2798
2799	/**
2800	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2801	* the VMCS.
2802	*
2803	* @returns VBox status code.
2804	* @param pVM The cross context VM structure.
2805	* @param pVCpu The cross context virtual CPU structure.
2806	*/
2807	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2808	{
2809	NOREF(pVM); NOREF(pVCpu);
2810
2811	RTCCUINTREG uReg = ASMGetCR0();
2812	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2813	AssertRCReturn(rc, rc);
2814
2815	uReg = ASMGetCR3();
2816	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2817	AssertRCReturn(rc, rc);
2818
2819	uReg = ASMGetCR4();
2820	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2821	AssertRCReturn(rc, rc);
2822	return rc;
2823	}
2824
2825
2826	#if HC_ARCH_BITS == 64
2827	/**
2828	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2829	* requirements. See hmR0VmxSaveHostSegmentRegs().
2830	*/
2831	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2832	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2833	{ \
2834	bool fValidSelector = true; \
2835	if ((selValue) & X86_SEL_LDT) \
2836	{ \
2837	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2838	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2839	} \
2840	if (fValidSelector) \
2841	{ \
2842	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2843	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2844	} \
2845	(selValue) = 0; \
2846	}
2847	#endif
2848
2849
2850	/**
2851	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2852	* the host-state area in the VMCS.
2853	*
2854	* @returns VBox status code.
2855	* @param pVM The cross context VM structure.
2856	* @param pVCpu The cross context virtual CPU structure.
2857	*/
2858	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2859	{
2860	int rc = VERR_INTERNAL_ERROR_5;
2861
2862	#if HC_ARCH_BITS == 64
2863	/*
2864	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2865	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2866	*/
2867	AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2868	("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2869	#endif
2870
2871	/*
2872	* Host DS, ES, FS and GS segment registers.
2873	*/
2874	#if HC_ARCH_BITS == 64
2875	RTSEL uSelDS = ASMGetDS();
2876	RTSEL uSelES = ASMGetES();
2877	RTSEL uSelFS = ASMGetFS();
2878	RTSEL uSelGS = ASMGetGS();
2879	#else
2880	RTSEL uSelDS = 0;
2881	RTSEL uSelES = 0;
2882	RTSEL uSelFS = 0;
2883	RTSEL uSelGS = 0;
2884	#endif
2885
2886	/* Recalculate which host-state bits need to be manually restored. */
2887	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2888
2889	/*
2890	* Host CS and SS segment registers.
2891	*/
2892	RTSEL uSelCS = ASMGetCS();
2893	RTSEL uSelSS = ASMGetSS();
2894
2895	/*
2896	* Host TR segment register.
2897	*/
2898	RTSEL uSelTR = ASMGetTR();
2899
2900	#if HC_ARCH_BITS == 64
2901	/*
2902	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2903	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2904	*/
2905	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2906	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2907	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2908	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2909	# undef VMXLOCAL_ADJUST_HOST_SEG
2910	#endif
2911
2912	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2913	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2914	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2915	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2916	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2917	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2918	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2919	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2920	Assert(uSelCS);
2921	Assert(uSelTR);
2922
2923	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2924	#if 0
2925	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2926	Assert(uSelSS != 0);
2927	#endif
2928
2929	/* Write these host selector fields into the host-state area in the VMCS. */
2930	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2931	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2932	#if HC_ARCH_BITS == 64
2933	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2934	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2935	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2936	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2937	#else
2938	NOREF(uSelDS);
2939	NOREF(uSelES);
2940	NOREF(uSelFS);
2941	NOREF(uSelGS);
2942	#endif
2943	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2944
2945	/*
2946	* Host GDTR and IDTR.
2947	*/
2948	RTGDTR Gdtr;
2949	RTIDTR Idtr;
2950	RT_ZERO(Gdtr);
2951	RT_ZERO(Idtr);
2952	ASMGetGDTR(&Gdtr);
2953	ASMGetIDTR(&Idtr);
2954	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2955	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2956
2957	#if HC_ARCH_BITS == 64
2958	/*
2959	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2960	* maximum limit (0xffff) on every VM-exit.
2961	*/
2962	if (Gdtr.cbGdt != 0xffff)
2963	{
2964	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2965	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2966	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2967	}
2968
2969	/*
2970	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2971	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2972	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2973	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2974	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2975	* hosts where we are pretty sure it won't cause trouble.
2976	*/
2977	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
2978	if (Idtr.cbIdt < 0x0fff)
2979	# else
2980	if (Idtr.cbIdt != 0xffff)
2981	# endif
2982	{
2983	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2984	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2985	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2986	}
2987	#endif
2988
2989	/*
2990	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2991	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2992	*/
2993	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
2994	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
2995	VERR_VMX_INVALID_HOST_STATE);
2996
2997	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2998	#if HC_ARCH_BITS == 64
2999	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3000
3001	/*
3002	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3003	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3004	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3005	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3006	*
3007	* [1] See Intel spec. 3.5 "System Descriptor Types".
3008	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3009	*/
3010	Assert(pDesc->System.u4Type == 11);
3011	if ( pDesc->System.u16LimitLow != 0x67
3012	\|\| pDesc->System.u4LimitHigh)
3013	{
3014	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3015	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3016	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3017	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3018	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3019
3020	/* Store the GDTR here as we need it while restoring TR. */
3021	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3022	}
3023	#else
3024	NOREF(pVM);
3025	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3026	#endif
3027	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3028	AssertRCReturn(rc, rc);
3029
3030	/*
3031	* Host FS base and GS base.
3032	*/
3033	#if HC_ARCH_BITS == 64
3034	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3035	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3036	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3037	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3038
3039	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3040	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3041	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3042	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3043	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3044	#endif
3045	return rc;
3046	}
3047
3048
3049	/**
3050	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3051	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3052	* the host after every successful VM-exit.
3053	*
3054	* @returns VBox status code.
3055	* @param pVM The cross context VM structure.
3056	* @param pVCpu The cross context virtual CPU structure.
3057	*
3058	* @remarks No-long-jump zone!!!
3059	*/
3060	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3061	{
3062	NOREF(pVM);
3063
3064	AssertPtr(pVCpu);
3065	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3066
3067	int rc = VINF_SUCCESS;
3068	#if HC_ARCH_BITS == 64
3069	if (pVM->hm.s.fAllow64BitGuests)
3070	hmR0VmxLazySaveHostMsrs(pVCpu);
3071	#endif
3072
3073	/*
3074	* Host Sysenter MSRs.
3075	*/
3076	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3077	AssertRCReturn(rc, rc);
3078	#if HC_ARCH_BITS == 32
3079	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3080	AssertRCReturn(rc, rc);
3081	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3082	#else
3083	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3084	AssertRCReturn(rc, rc);
3085	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3086	#endif
3087	AssertRCReturn(rc, rc);
3088
3089	/*
3090	* Host EFER MSR.
3091	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3092	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3093	*/
3094	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3095	{
3096	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3097	AssertRCReturn(rc, rc);
3098	}
3099
3100	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3101	* hmR0VmxLoadGuestExitCtls() !! */
3102
3103	return rc;
3104	}
3105
3106
3107	/**
3108	* Figures out if we need to swap the EFER MSR which is
3109	* particularly expensive.
3110	*
3111	* We check all relevant bits. For now, that's everything
3112	* besides LMA/LME, as these two bits are handled by VM-entry,
3113	* see hmR0VmxLoadGuestExitCtls() and
3114	* hmR0VMxLoadGuestEntryCtls().
3115	*
3116	* @returns true if we need to load guest EFER, false otherwise.
3117	* @param pVCpu The cross context virtual CPU structure.
3118	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3119	* out-of-sync. Make sure to update the required fields
3120	* before using them.
3121	*
3122	* @remarks Requires EFER, CR4.
3123	* @remarks No-long-jump zone!!!
3124	*/
3125	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3126	{
3127	#ifdef HMVMX_ALWAYS_SWAP_EFER
3128	return true;
3129	#endif
3130
3131	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3132	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3133	if (CPUMIsGuestInLongMode(pVCpu))
3134	return false;
3135	#endif
3136
3137	PVM pVM = pVCpu->CTX_SUFF(pVM);
3138	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3139	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3140
3141	/*
3142	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3143	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3144	*/
3145	if ( CPUMIsGuestInLongMode(pVCpu)
3146	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3147	{
3148	return true;
3149	}
3150
3151	/*
3152	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3153	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3154	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3155	*/
3156	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3157	&& (pMixedCtx->cr0 & X86_CR0_PG)
3158	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3159	{
3160	/* Assert that host is PAE capable. */
3161	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3162	return true;
3163	}
3164
3165	/** @todo Check the latest Intel spec. for any other bits,
3166	* like SMEP/SMAP? */
3167	return false;
3168	}
3169
3170
3171	/**
3172	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3173	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3174	* controls".
3175	*
3176	* @returns VBox status code.
3177	* @param pVCpu The cross context virtual CPU structure.
3178	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3179	* out-of-sync. Make sure to update the required fields
3180	* before using them.
3181	*
3182	* @remarks Requires EFER.
3183	* @remarks No-long-jump zone!!!
3184	*/
3185	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3186	{
3187	int rc = VINF_SUCCESS;
3188	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3189	{
3190	PVM pVM = pVCpu->CTX_SUFF(pVM);
3191	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3192	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3193
3194	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3195	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3196
3197	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3198	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3199	{
3200	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3201	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3202	}
3203	else
3204	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3205
3206	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3207	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3208	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3209	{
3210	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3211	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3212	}
3213
3214	/*
3215	* The following should -not- be set (since we're not in SMM mode):
3216	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3217	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3218	*/
3219
3220	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3221	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3222
3223	if ((val & zap) != val)
3224	{
3225	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3226	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3227	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3228	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3229	}
3230
3231	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3232	AssertRCReturn(rc, rc);
3233
3234	pVCpu->hm.s.vmx.u32EntryCtls = val;
3235	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3236	}
3237	return rc;
3238	}
3239
3240
3241	/**
3242	* Sets up the VM-exit controls in the VMCS.
3243	*
3244	* @returns VBox status code.
3245	* @param pVCpu The cross context virtual CPU structure.
3246	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3247	* out-of-sync. Make sure to update the required fields
3248	* before using them.
3249	*
3250	* @remarks Requires EFER.
3251	*/
3252	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3253	{
3254	NOREF(pMixedCtx);
3255
3256	int rc = VINF_SUCCESS;
3257	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3258	{
3259	PVM pVM = pVCpu->CTX_SUFF(pVM);
3260	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3261	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3262
3263	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3264	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3265
3266	/*
3267	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3268	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3269	*/
3270	#if HC_ARCH_BITS == 64
3271	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3272	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3273	#else
3274	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3275	{
3276	/* The switcher returns to long mode, EFER is managed by the switcher. */
3277	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3278	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3279	}
3280	else
3281	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3282	#endif
3283
3284	/* If the newer VMCS fields for managing EFER exists, use it. */
3285	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3286	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3287	{
3288	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3289	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3290	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3291	}
3292
3293	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3294	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3295
3296	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3297	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3298	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3299
3300	if ( pVM->hm.s.vmx.fUsePreemptTimer
3301	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3302	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3303
3304	if ((val & zap) != val)
3305	{
3306	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3307	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3308	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3309	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3310	}
3311
3312	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3313	AssertRCReturn(rc, rc);
3314
3315	pVCpu->hm.s.vmx.u32ExitCtls = val;
3316	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3317	}
3318	return rc;
3319	}
3320
3321
3322	/**
3323	* Loads the guest APIC and related state.
3324	*
3325	* @returns VBox status code.
3326	* @param pVCpu The cross context virtual CPU structure.
3327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3328	* out-of-sync. Make sure to update the required fields
3329	* before using them.
3330	*/
3331	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3332	{
3333	NOREF(pMixedCtx);
3334
3335	int rc = VINF_SUCCESS;
3336	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3337	{
3338	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3339	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3340	{
3341	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3342
3343	bool fPendingIntr = false;
3344	uint8_t u8Tpr = 0;
3345	uint8_t u8PendingIntr = 0;
3346	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3347	AssertRCReturn(rc, rc);
3348
3349	/*
3350	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3351	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3352	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3353	* the interrupt when we VM-exit for other reasons.
3354	*/
3355	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3356	uint32_t u32TprThreshold = 0;
3357	if (fPendingIntr)
3358	{
3359	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3360	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3361	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3362	if (u8PendingPriority <= u8TprPriority)
3363	u32TprThreshold = u8PendingPriority;
3364	else
3365	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3366	}
3367	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3368
3369	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3370	AssertRCReturn(rc, rc);
3371	}
3372
3373	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3374	}
3375	return rc;
3376	}
3377
3378
3379	/**
3380	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3381	*
3382	* @returns Guest's interruptibility-state.
3383	* @param pVCpu The cross context virtual CPU structure.
3384	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3385	* out-of-sync. Make sure to update the required fields
3386	* before using them.
3387	*
3388	* @remarks No-long-jump zone!!!
3389	*/
3390	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3391	{
3392	/*
3393	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3394	*/
3395	uint32_t uIntrState = 0;
3396	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3397	{
3398	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3399	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3400	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3401	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3402	{
3403	if (pMixedCtx->eflags.Bits.u1IF)
3404	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3405	else
3406	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3407	}
3408	/* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3409	}
3410
3411	/*
3412	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3413	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3414	* setting this would block host-NMIs and IRET will not clear the blocking.
3415	*
3416	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3417	*/
3418	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3419	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3420	{
3421	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3422	}
3423
3424	return uIntrState;
3425	}
3426
3427
3428	/**
3429	* Loads the guest's interruptibility-state into the guest-state area in the
3430	* VMCS.
3431	*
3432	* @returns VBox status code.
3433	* @param pVCpu The cross context virtual CPU structure.
3434	* @param uIntrState The interruptibility-state to set.
3435	*/
3436	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3437	{
3438	NOREF(pVCpu);
3439	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3440	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3441	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3442	AssertRC(rc);
3443	return rc;
3444	}
3445
3446
3447	/**
3448	* Loads the exception intercepts required for guest execution in the VMCS.
3449	*
3450	* @returns VBox status code.
3451	* @param pVCpu The cross context virtual CPU structure.
3452	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3453	* out-of-sync. Make sure to update the required fields
3454	* before using them.
3455	*/
3456	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3457	{
3458	NOREF(pMixedCtx);
3459	int rc = VINF_SUCCESS;
3460	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3461	{
3462	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3463	if (pVCpu->hm.s.fGIMTrapXcptUD)
3464	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3465	else
3466	{
3467	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3468	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3469	#endif
3470	}
3471
3472	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3473	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3474
3475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3476	AssertRCReturn(rc, rc);
3477
3478	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3479	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3480	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3481	}
3482	return rc;
3483	}
3484
3485
3486	/**
3487	* Loads the guest's RIP into the guest-state area in the VMCS.
3488	*
3489	* @returns VBox status code.
3490	* @param pVCpu The cross context virtual CPU structure.
3491	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3492	* out-of-sync. Make sure to update the required fields
3493	* before using them.
3494	*
3495	* @remarks No-long-jump zone!!!
3496	*/
3497	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3498	{
3499	int rc = VINF_SUCCESS;
3500	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3501	{
3502	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3503	AssertRCReturn(rc, rc);
3504
3505	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3506	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3507	HMCPU_CF_VALUE(pVCpu)));
3508	}
3509	return rc;
3510	}
3511
3512
3513	/**
3514	* Loads the guest's RSP into the guest-state area in the VMCS.
3515	*
3516	* @returns VBox status code.
3517	* @param pVCpu The cross context virtual CPU structure.
3518	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3519	* out-of-sync. Make sure to update the required fields
3520	* before using them.
3521	*
3522	* @remarks No-long-jump zone!!!
3523	*/
3524	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3525	{
3526	int rc = VINF_SUCCESS;
3527	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3528	{
3529	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3530	AssertRCReturn(rc, rc);
3531
3532	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3533	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3534	}
3535	return rc;
3536	}
3537
3538
3539	/**
3540	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3541	*
3542	* @returns VBox status code.
3543	* @param pVCpu The cross context virtual CPU structure.
3544	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3545	* out-of-sync. Make sure to update the required fields
3546	* before using them.
3547	*
3548	* @remarks No-long-jump zone!!!
3549	*/
3550	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3551	{
3552	int rc = VINF_SUCCESS;
3553	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3554	{
3555	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3556	Let us assert it as such and use 32-bit VMWRITE. */
3557	Assert(!(pMixedCtx->rflags.u64 >> 32));
3558	X86EFLAGS Eflags = pMixedCtx->eflags;
3559	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3560	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3561	* These will never be cleared/set, unless some other part of the VMM
3562	* code is buggy - in which case we're better of finding and fixing
3563	* those bugs than hiding them. */
3564	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3565	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3566	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3567	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3568
3569	/*
3570	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3571	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3572	*/
3573	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3574	{
3575	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3576	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3577	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3578	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3579	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3580	}
3581
3582	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3583	AssertRCReturn(rc, rc);
3584
3585	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3586	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3587	}
3588	return rc;
3589	}
3590
3591
3592	/**
3593	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3594	*
3595	* @returns VBox status code.
3596	* @param pVCpu The cross context virtual CPU structure.
3597	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3598	* out-of-sync. Make sure to update the required fields
3599	* before using them.
3600	*
3601	* @remarks No-long-jump zone!!!
3602	*/
3603	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3604	{
3605	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3606	AssertRCReturn(rc, rc);
3607	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3608	AssertRCReturn(rc, rc);
3609	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3610	AssertRCReturn(rc, rc);
3611	return rc;
3612	}
3613
3614
3615	/**
3616	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3617	* CR0 is partially shared with the host and we have to consider the FPU bits.
3618	*
3619	* @returns VBox status code.
3620	* @param pVCpu The cross context virtual CPU structure.
3621	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3622	* out-of-sync. Make sure to update the required fields
3623	* before using them.
3624	*
3625	* @remarks No-long-jump zone!!!
3626	*/
3627	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3628	{
3629	/*
3630	* Guest CR0.
3631	* Guest FPU.
3632	*/
3633	int rc = VINF_SUCCESS;
3634	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3635	{
3636	Assert(!(pMixedCtx->cr0 >> 32));
3637	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3638	PVM pVM = pVCpu->CTX_SUFF(pVM);
3639
3640	/* The guest's view (read access) of its CR0 is unblemished. */
3641	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3642	AssertRCReturn(rc, rc);
3643	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3644
3645	/* Setup VT-x's view of the guest CR0. */
3646	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3647	if (pVM->hm.s.fNestedPaging)
3648	{
3649	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3650	{
3651	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3652	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3653	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3654	}
3655	else
3656	{
3657	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3658	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3659	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3660	}
3661
3662	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3663	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3664	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3665
3666	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3667	AssertRCReturn(rc, rc);
3668	}
3669	else
3670	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3671
3672	/*
3673	* Guest FPU bits.
3674	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3675	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3676	*/
3677	u32GuestCR0 \|= X86_CR0_NE;
3678	bool fInterceptNM = false;
3679	if (CPUMIsGuestFPUStateActive(pVCpu))
3680	{
3681	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3682	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3683	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3684	}
3685	else
3686	{
3687	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3688	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3689	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3690	}
3691
3692	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3693	bool fInterceptMF = false;
3694	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3695	fInterceptMF = true;
3696
3697	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3698	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3699	{
3700	Assert(PDMVmmDevHeapIsEnabled(pVM));
3701	Assert(pVM->hm.s.vmx.pRealModeTSS);
3702	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3703	fInterceptNM = true;
3704	fInterceptMF = true;
3705	}
3706	else
3707	{
3708	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3709	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3710	}
3711	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3712
3713	if (fInterceptNM)
3714	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3715	else
3716	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3717
3718	if (fInterceptMF)
3719	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3720	else
3721	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3722
3723	/* Additional intercepts for debugging, define these yourself explicitly. */
3724	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3725	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3726	\| RT_BIT(X86_XCPT_BP)
3727	\| RT_BIT(X86_XCPT_DE)
3728	\| RT_BIT(X86_XCPT_NM)
3729	\| RT_BIT(X86_XCPT_TS)
3730	\| RT_BIT(X86_XCPT_UD)
3731	\| RT_BIT(X86_XCPT_NP)
3732	\| RT_BIT(X86_XCPT_SS)
3733	\| RT_BIT(X86_XCPT_GP)
3734	\| RT_BIT(X86_XCPT_PF)
3735	\| RT_BIT(X86_XCPT_MF)
3736	;
3737	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3738	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3739	#endif
3740
3741	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3742
3743	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3744	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3745	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3746	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3747	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3748	else
3749	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3750
3751	u32GuestCR0 \|= uSetCR0;
3752	u32GuestCR0 &= uZapCR0;
3753	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3754
3755	/* Write VT-x's view of the guest CR0 into the VMCS. */
3756	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3757	AssertRCReturn(rc, rc);
3758	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3759	uZapCR0));
3760
3761	/*
3762	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3763	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3764	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3765	*/
3766	uint32_t u32CR0Mask = 0;
3767	u32CR0Mask = X86_CR0_PE
3768	\| X86_CR0_NE
3769	\| X86_CR0_WP
3770	\| X86_CR0_PG
3771	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3772	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3773	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3774
3775	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3776	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3777	* and @bugref{6944}. */
3778	#if 0
3779	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3780	u32CR0Mask &= ~X86_CR0_PE;
3781	#endif
3782	if (pVM->hm.s.fNestedPaging)
3783	u32CR0Mask &= ~X86_CR0_WP;
3784
3785	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3786	if (fInterceptNM)
3787	{
3788	u32CR0Mask \|= X86_CR0_TS
3789	\| X86_CR0_MP;
3790	}
3791
3792	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3793	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3794	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3795	AssertRCReturn(rc, rc);
3796	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3797
3798	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3799	}
3800	return rc;
3801	}
3802
3803
3804	/**
3805	* Loads the guest control registers (CR3, CR4) into the guest-state area
3806	* in the VMCS.
3807	*
3808	* @returns VBox status code.
3809	* @param pVCpu The cross context virtual CPU structure.
3810	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3811	* out-of-sync. Make sure to update the required fields
3812	* before using them.
3813	*
3814	* @remarks No-long-jump zone!!!
3815	*/
3816	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3817	{
3818	int rc = VINF_SUCCESS;
3819	PVM pVM = pVCpu->CTX_SUFF(pVM);
3820
3821	/*
3822	* Guest CR2.
3823	* It's always loaded in the assembler code. Nothing to do here.
3824	*/
3825
3826	/*
3827	* Guest CR3.
3828	*/
3829	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3830	{
3831	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3832	if (pVM->hm.s.fNestedPaging)
3833	{
3834	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3835
3836	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3837	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3838	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3839	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3840
3841	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3842	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3843	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3844
3845	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3846	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3847	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3848	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3849	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3850	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3851	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3852
3853	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3854	AssertRCReturn(rc, rc);
3855	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3856
3857	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3858	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3859	{
3860	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3861	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3862	{
3863	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3864	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3865	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3866	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3867	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3868	}
3869
3870	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3871	have Unrestricted Execution to handle the guest when it's not using paging. */
3872	GCPhysGuestCR3 = pMixedCtx->cr3;
3873	}
3874	else
3875	{
3876	/*
3877	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3878	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3879	* EPT takes care of translating it to host-physical addresses.
3880	*/
3881	RTGCPHYS GCPhys;
3882	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3883	Assert(PDMVmmDevHeapIsEnabled(pVM));
3884
3885	/* We obtain it here every time as the guest could have relocated this PCI region. */
3886	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3887	AssertRCReturn(rc, rc);
3888
3889	GCPhysGuestCR3 = GCPhys;
3890	}
3891
3892	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3893	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3894	}
3895	else
3896	{
3897	/* Non-nested paging case, just use the hypervisor's CR3. */
3898	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3899
3900	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3901	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3902	}
3903	AssertRCReturn(rc, rc);
3904
3905	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3906	}
3907
3908	/*
3909	* Guest CR4.
3910	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
3911	*/
3912	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3913	{
3914	Assert(!(pMixedCtx->cr4 >> 32));
3915	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3916
3917	/* The guest's view of its CR4 is unblemished. */
3918	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3919	AssertRCReturn(rc, rc);
3920	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3921
3922	/* Setup VT-x's view of the guest CR4. */
3923	/*
3924	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3925	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3926	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3927	*/
3928	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3929	{
3930	Assert(pVM->hm.s.vmx.pRealModeTSS);
3931	Assert(PDMVmmDevHeapIsEnabled(pVM));
3932	u32GuestCR4 &= ~X86_CR4_VME;
3933	}
3934
3935	if (pVM->hm.s.fNestedPaging)
3936	{
3937	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3938	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3939	{
3940	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3941	u32GuestCR4 \|= X86_CR4_PSE;
3942	/* Our identity mapping is a 32-bit page directory. */
3943	u32GuestCR4 &= ~X86_CR4_PAE;
3944	}
3945	/* else use guest CR4.*/
3946	}
3947	else
3948	{
3949	/*
3950	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3951	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3952	*/
3953	switch (pVCpu->hm.s.enmShadowMode)
3954	{
3955	case PGMMODE_REAL: /* Real-mode. */
3956	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3957	case PGMMODE_32_BIT: /* 32-bit paging. */
3958	{
3959	u32GuestCR4 &= ~X86_CR4_PAE;
3960	break;
3961	}
3962
3963	case PGMMODE_PAE: /* PAE paging. */
3964	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3965	{
3966	u32GuestCR4 \|= X86_CR4_PAE;
3967	break;
3968	}
3969
3970	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3971	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3972	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3973	break;
3974	#endif
3975	default:
3976	AssertFailed();
3977	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3978	}
3979	}
3980
3981	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3982	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3983	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3984	u32GuestCR4 \|= uSetCR4;
3985	u32GuestCR4 &= uZapCR4;
3986
3987	/* Write VT-x's view of the guest CR4 into the VMCS. */
3988	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
3989	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3990	AssertRCReturn(rc, rc);
3991
3992	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3993	uint32_t u32CR4Mask = X86_CR4_VME
3994	\| X86_CR4_PAE
3995	\| X86_CR4_PGE
3996	\| X86_CR4_PSE
3997	\| X86_CR4_VMXE;
3998	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
3999	u32CR4Mask \|= X86_CR4_OSXSAVE;
4000	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4001	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4002	AssertRCReturn(rc, rc);
4003
4004	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4005	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4006
4007	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4008	}
4009	return rc;
4010	}
4011
4012
4013	/**
4014	* Loads the guest debug registers into the guest-state area in the VMCS.
4015	*
4016	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4017	*
4018	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4019	*
4020	* @returns VBox status code.
4021	* @param pVCpu The cross context virtual CPU structure.
4022	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4023	* out-of-sync. Make sure to update the required fields
4024	* before using them.
4025	*
4026	* @remarks No-long-jump zone!!!
4027	*/
4028	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4029	{
4030	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4031	return VINF_SUCCESS;
4032
4033	#ifdef VBOX_STRICT
4034	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4035	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4036	{
4037	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4038	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4039	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4040	}
4041	#endif
4042
4043	int rc;
4044	PVM pVM = pVCpu->CTX_SUFF(pVM);
4045	bool fSteppingDB = false;
4046	bool fInterceptMovDRx = false;
4047	if (pVCpu->hm.s.fSingleInstruction)
4048	{
4049	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4050	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4051	{
4052	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4053	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4054	AssertRCReturn(rc, rc);
4055	Assert(fSteppingDB == false);
4056	}
4057	else
4058	{
4059	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4060	pVCpu->hm.s.fClearTrapFlag = true;
4061	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4062	fSteppingDB = true;
4063	}
4064	}
4065
4066	if ( fSteppingDB
4067	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4068	{
4069	/*
4070	* Use the combined guest and host DRx values found in the hypervisor
4071	* register set because the debugger has breakpoints active or someone
4072	* is single stepping on the host side without a monitor trap flag.
4073	*
4074	* Note! DBGF expects a clean DR6 state before executing guest code.
4075	*/
4076	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4077	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4078	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4079	{
4080	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4081	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4082	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4083	}
4084	else
4085	#endif
4086	if (!CPUMIsHyperDebugStateActive(pVCpu))
4087	{
4088	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4089	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4090	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4091	}
4092
4093	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4094	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4095	AssertRCReturn(rc, rc);
4096
4097	pVCpu->hm.s.fUsingHyperDR7 = true;
4098	fInterceptMovDRx = true;
4099	}
4100	else
4101	{
4102	/*
4103	* If the guest has enabled debug registers, we need to load them prior to
4104	* executing guest code so they'll trigger at the right time.
4105	*/
4106	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4107	{
4108	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4109	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4110	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4111	{
4112	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4113	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4114	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4115	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4116	}
4117	else
4118	#endif
4119	if (!CPUMIsGuestDebugStateActive(pVCpu))
4120	{
4121	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4122	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4123	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4124	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4125	}
4126	Assert(!fInterceptMovDRx);
4127	}
4128	/*
4129	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4130	* must intercept #DB in order to maintain a correct DR6 guest value, and
4131	* because we need to intercept it to prevent nested #DBs from hanging the
4132	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4133	*/
4134	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4135	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4136	&& !CPUMIsGuestDebugStateActive(pVCpu))
4137	#else
4138	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4139	#endif
4140	{
4141	fInterceptMovDRx = true;
4142	}
4143
4144	/* Update guest DR7. */
4145	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4146	AssertRCReturn(rc, rc);
4147
4148	pVCpu->hm.s.fUsingHyperDR7 = false;
4149	}
4150
4151	/*
4152	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4153	*/
4154	if (fInterceptMovDRx)
4155	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4156	else
4157	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4158	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4159	AssertRCReturn(rc, rc);
4160
4161	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4162	return VINF_SUCCESS;
4163	}
4164
4165
4166	#ifdef VBOX_STRICT
4167	/**
4168	* Strict function to validate segment registers.
4169	*
4170	* @remarks ASSUMES CR0 is up to date.
4171	*/
4172	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4173	{
4174	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4175	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4176	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4177	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4178	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4179	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4180	{
4181	/* Protected mode checks */
4182	/* CS */
4183	Assert(pCtx->cs.Attr.n.u1Present);
4184	Assert(!(pCtx->cs.Attr.u & 0xf00));
4185	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4186	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4187	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4188	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4189	\|\| (pCtx->cs.Attr.n.u1Granularity));
4190	/* CS cannot be loaded with NULL in protected mode. */
4191	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4192	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4193	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4194	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4195	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4196	else
4197	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4198	/* SS */
4199	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4200	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4201	if ( !(pCtx->cr0 & X86_CR0_PE)
4202	\|\| pCtx->cs.Attr.n.u4Type == 3)
4203	{
4204	Assert(!pCtx->ss.Attr.n.u2Dpl);
4205	}
4206	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4207	{
4208	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4209	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4210	Assert(pCtx->ss.Attr.n.u1Present);
4211	Assert(!(pCtx->ss.Attr.u & 0xf00));
4212	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4213	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4214	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4215	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4216	\|\| (pCtx->ss.Attr.n.u1Granularity));
4217	}
4218	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4219	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4220	{
4221	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4222	Assert(pCtx->ds.Attr.n.u1Present);
4223	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4224	Assert(!(pCtx->ds.Attr.u & 0xf00));
4225	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4226	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4227	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4228	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4229	\|\| (pCtx->ds.Attr.n.u1Granularity));
4230	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4231	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4232	}
4233	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4234	{
4235	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4236	Assert(pCtx->es.Attr.n.u1Present);
4237	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4238	Assert(!(pCtx->es.Attr.u & 0xf00));
4239	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4240	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4241	\|\| !(pCtx->es.Attr.n.u1Granularity));
4242	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4243	\|\| (pCtx->es.Attr.n.u1Granularity));
4244	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4245	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4246	}
4247	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4248	{
4249	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4250	Assert(pCtx->fs.Attr.n.u1Present);
4251	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4252	Assert(!(pCtx->fs.Attr.u & 0xf00));
4253	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4254	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4255	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4256	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4257	\|\| (pCtx->fs.Attr.n.u1Granularity));
4258	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4259	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4260	}
4261	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4262	{
4263	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4264	Assert(pCtx->gs.Attr.n.u1Present);
4265	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4266	Assert(!(pCtx->gs.Attr.u & 0xf00));
4267	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4268	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4269	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4270	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4271	\|\| (pCtx->gs.Attr.n.u1Granularity));
4272	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4273	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4274	}
4275	/* 64-bit capable CPUs. */
4276	# if HC_ARCH_BITS == 64
4277	Assert(!(pCtx->cs.u64Base >> 32));
4278	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4279	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4280	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4281	# endif
4282	}
4283	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4284	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4285	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4286	{
4287	/* Real and v86 mode checks. */
4288	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4289	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4290	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4291	{
4292	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4293	}
4294	else
4295	{
4296	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4297	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4298	}
4299
4300	/* CS */
4301	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4302	Assert(pCtx->cs.u32Limit == 0xffff);
4303	Assert(u32CSAttr == 0xf3);
4304	/* SS */
4305	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4306	Assert(pCtx->ss.u32Limit == 0xffff);
4307	Assert(u32SSAttr == 0xf3);
4308	/* DS */
4309	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4310	Assert(pCtx->ds.u32Limit == 0xffff);
4311	Assert(u32DSAttr == 0xf3);
4312	/* ES */
4313	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4314	Assert(pCtx->es.u32Limit == 0xffff);
4315	Assert(u32ESAttr == 0xf3);
4316	/* FS */
4317	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4318	Assert(pCtx->fs.u32Limit == 0xffff);
4319	Assert(u32FSAttr == 0xf3);
4320	/* GS */
4321	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4322	Assert(pCtx->gs.u32Limit == 0xffff);
4323	Assert(u32GSAttr == 0xf3);
4324	/* 64-bit capable CPUs. */
4325	# if HC_ARCH_BITS == 64
4326	Assert(!(pCtx->cs.u64Base >> 32));
4327	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4328	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4329	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4330	# endif
4331	}
4332	}
4333	#endif /* VBOX_STRICT */
4334
4335
4336	/**
4337	* Writes a guest segment register into the guest-state area in the VMCS.
4338	*
4339	* @returns VBox status code.
4340	* @param pVCpu The cross context virtual CPU structure.
4341	* @param idxSel Index of the selector in the VMCS.
4342	* @param idxLimit Index of the segment limit in the VMCS.
4343	* @param idxBase Index of the segment base in the VMCS.
4344	* @param idxAccess Index of the access rights of the segment in the VMCS.
4345	* @param pSelReg Pointer to the segment selector.
4346	*
4347	* @remarks No-long-jump zone!!!
4348	*/
4349	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4350	uint32_t idxAccess, PCPUMSELREG pSelReg)
4351	{
4352	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4353	AssertRCReturn(rc, rc);
4354	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4355	AssertRCReturn(rc, rc);
4356	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4357	AssertRCReturn(rc, rc);
4358
4359	uint32_t u32Access = pSelReg->Attr.u;
4360	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4361	{
4362	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4363	u32Access = 0xf3;
4364	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4365	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4366	}
4367	else
4368	{
4369	/*
4370	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4371	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4372	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4373	* loaded in protected-mode have their attribute as 0.
4374	*/
4375	if (!u32Access)
4376	u32Access = X86DESCATTR_UNUSABLE;
4377	}
4378
4379	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4380	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4381	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4382
4383	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4384	AssertRCReturn(rc, rc);
4385	return rc;
4386	}
4387
4388
4389	/**
4390	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4391	* into the guest-state area in the VMCS.
4392	*
4393	* @returns VBox status code.
4394	* @param pVCpu The cross context virtual CPU structure.
4395	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4396	* out-of-sync. Make sure to update the required fields
4397	* before using them.
4398	*
4399	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4400	* @remarks No-long-jump zone!!!
4401	*/
4402	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4403	{
4404	int rc = VERR_INTERNAL_ERROR_5;
4405	PVM pVM = pVCpu->CTX_SUFF(pVM);
4406
4407	/*
4408	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4409	*/
4410	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4411	{
4412	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4413	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4414	{
4415	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4416	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4417	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4418	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4419	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4420	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4421	}
4422
4423	#ifdef VBOX_WITH_REM
4424	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4425	{
4426	Assert(pVM->hm.s.vmx.pRealModeTSS);
4427	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4428	if ( pVCpu->hm.s.vmx.fWasInRealMode
4429	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4430	{
4431	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4432	in real-mode (e.g. OpenBSD 4.0) */
4433	REMFlushTBs(pVM);
4434	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4435	pVCpu->hm.s.vmx.fWasInRealMode = false;
4436	}
4437	}
4438	#endif
4439	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4440	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4441	AssertRCReturn(rc, rc);
4442	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4443	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4444	AssertRCReturn(rc, rc);
4445	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4446	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4447	AssertRCReturn(rc, rc);
4448	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4449	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4450	AssertRCReturn(rc, rc);
4451	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4452	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4453	AssertRCReturn(rc, rc);
4454	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4455	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4456	AssertRCReturn(rc, rc);
4457
4458	#ifdef VBOX_STRICT
4459	/* Validate. */
4460	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4461	#endif
4462
4463	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4464	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4465	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4466	}
4467
4468	/*
4469	* Guest TR.
4470	*/
4471	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4472	{
4473	/*
4474	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4475	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4476	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4477	*/
4478	uint16_t u16Sel = 0;
4479	uint32_t u32Limit = 0;
4480	uint64_t u64Base = 0;
4481	uint32_t u32AccessRights = 0;
4482
4483	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4484	{
4485	u16Sel = pMixedCtx->tr.Sel;
4486	u32Limit = pMixedCtx->tr.u32Limit;
4487	u64Base = pMixedCtx->tr.u64Base;
4488	u32AccessRights = pMixedCtx->tr.Attr.u;
4489	}
4490	else
4491	{
4492	Assert(pVM->hm.s.vmx.pRealModeTSS);
4493	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4494
4495	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4496	RTGCPHYS GCPhys;
4497	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4498	AssertRCReturn(rc, rc);
4499
4500	X86DESCATTR DescAttr;
4501	DescAttr.u = 0;
4502	DescAttr.n.u1Present = 1;
4503	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4504
4505	u16Sel = 0;
4506	u32Limit = HM_VTX_TSS_SIZE;
4507	u64Base = GCPhys; /* in real-mode phys = virt. */
4508	u32AccessRights = DescAttr.u;
4509	}
4510
4511	/* Validate. */
4512	Assert(!(u16Sel & RT_BIT(2)));
4513	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4514	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4515	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4516	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4517	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4518	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4519	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4520	Assert( (u32Limit & 0xfff) == 0xfff
4521	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4522	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4523	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4524
4525	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4526	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4527	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4528	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4529
4530	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4531	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4532	}
4533
4534	/*
4535	* Guest GDTR.
4536	*/
4537	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4538	{
4539	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4540	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4541
4542	/* Validate. */
4543	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4544
4545	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4546	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4547	}
4548
4549	/*
4550	* Guest LDTR.
4551	*/
4552	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4553	{
4554	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4555	uint32_t u32Access = 0;
4556	if (!pMixedCtx->ldtr.Attr.u)
4557	u32Access = X86DESCATTR_UNUSABLE;
4558	else
4559	u32Access = pMixedCtx->ldtr.Attr.u;
4560
4561	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4562	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4563	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4564	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4565
4566	/* Validate. */
4567	if (!(u32Access & X86DESCATTR_UNUSABLE))
4568	{
4569	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4570	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4571	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4572	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4573	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4574	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4575	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4576	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4577	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4578	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4579	}
4580
4581	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4582	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4583	}
4584
4585	/*
4586	* Guest IDTR.
4587	*/
4588	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4589	{
4590	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4591	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4592
4593	/* Validate. */
4594	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4595
4596	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4597	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4598	}
4599
4600	return VINF_SUCCESS;
4601	}
4602
4603
4604	/**
4605	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4606	* areas.
4607	*
4608	* These MSRs will automatically be loaded to the host CPU on every successful
4609	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4610	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4611	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4612	*
4613	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4614	*
4615	* @returns VBox status code.
4616	* @param pVCpu The cross context virtual CPU structure.
4617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4618	* out-of-sync. Make sure to update the required fields
4619	* before using them.
4620	*
4621	* @remarks No-long-jump zone!!!
4622	*/
4623	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4624	{
4625	AssertPtr(pVCpu);
4626	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4627
4628	/*
4629	* MSRs that we use the auto-load/store MSR area in the VMCS.
4630	*/
4631	PVM pVM = pVCpu->CTX_SUFF(pVM);
4632	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4633	{
4634	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4635	#if HC_ARCH_BITS == 32
4636	if (pVM->hm.s.fAllow64BitGuests)
4637	{
4638	int rc = VINF_SUCCESS;
4639	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4640	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4641	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4642	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4643	AssertRCReturn(rc, rc);
4644	# ifdef LOG_ENABLED
4645	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4646	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4647	{
4648	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4649	pMsr->u64Value));
4650	}
4651	# endif
4652	}
4653	#endif
4654	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4655	}
4656
4657	/*
4658	* Guest Sysenter MSRs.
4659	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4660	* VM-exits on WRMSRs for these MSRs.
4661	*/
4662	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4663	{
4664	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4665	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4666	}
4667
4668	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4669	{
4670	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4671	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4672	}
4673
4674	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4675	{
4676	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4677	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4678	}
4679
4680	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4681	{
4682	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4683	{
4684	/*
4685	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4686	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4687	*/
4688	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4689	{
4690	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4691	AssertRCReturn(rc,rc);
4692	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4693	}
4694	else
4695	{
4696	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4697	NULL /* pfAddedAndUpdated */);
4698	AssertRCReturn(rc, rc);
4699
4700	/* We need to intercept reads too, see @bugref{7386#c16}. */
4701	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4702	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4703	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4704	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4705	}
4706	}
4707	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4708	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4709	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4710	}
4711
4712	return VINF_SUCCESS;
4713	}
4714
4715
4716	/**
4717	* Loads the guest activity state into the guest-state area in the VMCS.
4718	*
4719	* @returns VBox status code.
4720	* @param pVCpu The cross context virtual CPU structure.
4721	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4722	* out-of-sync. Make sure to update the required fields
4723	* before using them.
4724	*
4725	* @remarks No-long-jump zone!!!
4726	*/
4727	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4728	{
4729	NOREF(pMixedCtx);
4730	/** @todo See if we can make use of other states, e.g.
4731	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4732	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4733	{
4734	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4735	AssertRCReturn(rc, rc);
4736
4737	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4738	}
4739	return VINF_SUCCESS;
4740	}
4741
4742
4743	/**
4744	* Sets up the appropriate function to run guest code.
4745	*
4746	* @returns VBox status code.
4747	* @param pVCpu The cross context virtual CPU structure.
4748	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4749	* out-of-sync. Make sure to update the required fields
4750	* before using them.
4751	*
4752	* @remarks No-long-jump zone!!!
4753	*/
4754	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4755	{
4756	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4757	{
4758	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4759	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4760	#endif
4761	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4762	#if HC_ARCH_BITS == 32
4763	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4764	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4765	{
4766	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4767	{
4768	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4769	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4770	\| HM_CHANGED_VMX_ENTRY_CTLS
4771	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4772	}
4773	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4774	}
4775	#else
4776	/* 64-bit host. */
4777	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4778	#endif
4779	}
4780	else
4781	{
4782	/* Guest is not in long mode, use the 32-bit handler. */
4783	#if HC_ARCH_BITS == 32
4784	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4785	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4786	{
4787	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4788	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4789	\| HM_CHANGED_VMX_ENTRY_CTLS
4790	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4791	}
4792	#endif
4793	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4794	}
4795	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4796	return VINF_SUCCESS;
4797	}
4798
4799
4800	/**
4801	* Wrapper for running the guest code in VT-x.
4802	*
4803	* @returns VBox status code, no informational status codes.
4804	* @param pVM The cross context VM structure.
4805	* @param pVCpu The cross context virtual CPU structure.
4806	* @param pCtx Pointer to the guest-CPU context.
4807	*
4808	* @remarks No-long-jump zone!!!
4809	*/
4810	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4811	{
4812	/*
4813	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4814	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4815	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4816	*/
4817	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4818	/** @todo Add stats for resume vs launch. */
4819	#ifdef VBOX_WITH_KERNEL_USING_XMM
4820	int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4821	#else
4822	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4823	#endif
4824	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4825	return rc;
4826	}
4827
4828
4829	/**
4830	* Reports world-switch error and dumps some useful debug info.
4831	*
4832	* @param pVM The cross context VM structure.
4833	* @param pVCpu The cross context virtual CPU structure.
4834	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4835	* @param pCtx Pointer to the guest-CPU context.
4836	* @param pVmxTransient Pointer to the VMX transient structure (only
4837	* exitReason updated).
4838	*/
4839	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4840	{
4841	Assert(pVM);
4842	Assert(pVCpu);
4843	Assert(pCtx);
4844	Assert(pVmxTransient);
4845	HMVMX_ASSERT_PREEMPT_SAFE();
4846
4847	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4848	switch (rcVMRun)
4849	{
4850	case VERR_VMX_INVALID_VMXON_PTR:
4851	AssertFailed();
4852	break;
4853	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4854	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4855	{
4856	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4857	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4858	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4859	AssertRC(rc);
4860
4861	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4862	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4863	Cannot do it here as we may have been long preempted. */
4864
4865	#ifdef VBOX_STRICT
4866	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4867	pVmxTransient->uExitReason));
4868	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4869	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4870	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4871	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4872	else
4873	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4874	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4875	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4876
4877	/* VMX control bits. */
4878	uint32_t u32Val;
4879	uint64_t u64Val;
4880	RTHCUINTREG uHCReg;
4881	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4882	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4883	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4884	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4885	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4886	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4887	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4888	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4889	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4890	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4891	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4892	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4893	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4894	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4895	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4896	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4897	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4898	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4899	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4900	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4901	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4902	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4903	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4904	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4905	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4906	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4907	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4908	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4909	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4910	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4911	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4912	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4913	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4914	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4915	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4916	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4917	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4918	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4919	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4920	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4921	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4922	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4923
4924	/* Guest bits. */
4925	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4926	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4927	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4928	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4929	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4930	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4931	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4932	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4933
4934	/* Host bits. */
4935	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4936	Log4(("Host CR0 %#RHr\n", uHCReg));
4937	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4938	Log4(("Host CR3 %#RHr\n", uHCReg));
4939	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4940	Log4(("Host CR4 %#RHr\n", uHCReg));
4941
4942	RTGDTR HostGdtr;
4943	PCX86DESCHC pDesc;
4944	ASMGetGDTR(&HostGdtr);
4945	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4946	Log4(("Host CS %#08x\n", u32Val));
4947	if (u32Val < HostGdtr.cbGdt)
4948	{
4949	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4950	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4951	}
4952
4953	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4954	Log4(("Host DS %#08x\n", u32Val));
4955	if (u32Val < HostGdtr.cbGdt)
4956	{
4957	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4958	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4959	}
4960
4961	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4962	Log4(("Host ES %#08x\n", u32Val));
4963	if (u32Val < HostGdtr.cbGdt)
4964	{
4965	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4966	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4967	}
4968
4969	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4970	Log4(("Host FS %#08x\n", u32Val));
4971	if (u32Val < HostGdtr.cbGdt)
4972	{
4973	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4974	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4975	}
4976
4977	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4978	Log4(("Host GS %#08x\n", u32Val));
4979	if (u32Val < HostGdtr.cbGdt)
4980	{
4981	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4982	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4983	}
4984
4985	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4986	Log4(("Host SS %#08x\n", u32Val));
4987	if (u32Val < HostGdtr.cbGdt)
4988	{
4989	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4990	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4991	}
4992
4993	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4994	Log4(("Host TR %#08x\n", u32Val));
4995	if (u32Val < HostGdtr.cbGdt)
4996	{
4997	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4998	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4999	}
5000
5001	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5002	Log4(("Host TR Base %#RHv\n", uHCReg));
5003	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5004	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5005	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5006	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5007	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5008	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5009	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5010	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5011	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5012	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5013	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5014	Log4(("Host RSP %#RHv\n", uHCReg));
5015	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5016	Log4(("Host RIP %#RHv\n", uHCReg));
5017	# if HC_ARCH_BITS == 64
5018	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5019	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5020	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5021	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5022	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5023	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5024	# endif
5025	#endif /* VBOX_STRICT */
5026	break;
5027	}
5028
5029	default:
5030	/* Impossible */
5031	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5032	break;
5033	}
5034	NOREF(pVM); NOREF(pCtx);
5035	}
5036
5037
5038	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5039	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5040	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5041	#endif
5042	#ifdef VBOX_STRICT
5043	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5044	{
5045	switch (idxField)
5046	{
5047	case VMX_VMCS_GUEST_RIP:
5048	case VMX_VMCS_GUEST_RSP:
5049	case VMX_VMCS_GUEST_SYSENTER_EIP:
5050	case VMX_VMCS_GUEST_SYSENTER_ESP:
5051	case VMX_VMCS_GUEST_GDTR_BASE:
5052	case VMX_VMCS_GUEST_IDTR_BASE:
5053	case VMX_VMCS_GUEST_CS_BASE:
5054	case VMX_VMCS_GUEST_DS_BASE:
5055	case VMX_VMCS_GUEST_ES_BASE:
5056	case VMX_VMCS_GUEST_FS_BASE:
5057	case VMX_VMCS_GUEST_GS_BASE:
5058	case VMX_VMCS_GUEST_SS_BASE:
5059	case VMX_VMCS_GUEST_LDTR_BASE:
5060	case VMX_VMCS_GUEST_TR_BASE:
5061	case VMX_VMCS_GUEST_CR3:
5062	return true;
5063	}
5064	return false;
5065	}
5066
5067	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5068	{
5069	switch (idxField)
5070	{
5071	/* Read-only fields. */
5072	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5073	return true;
5074	}
5075	/* Remaining readable fields should also be writable. */
5076	return hmR0VmxIsValidWriteField(idxField);
5077	}
5078	#endif /* VBOX_STRICT */
5079
5080
5081	/**
5082	* Executes the specified handler in 64-bit mode.
5083	*
5084	* @returns VBox status code (no informational status codes).
5085	* @param pVM The cross context VM structure.
5086	* @param pVCpu The cross context virtual CPU structure.
5087	* @param pCtx Pointer to the guest CPU context.
5088	* @param enmOp The operation to perform.
5089	* @param cParams Number of parameters.
5090	* @param paParam Array of 32-bit parameters.
5091	*/
5092	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5093	uint32_t cParams, uint32_t *paParam)
5094	{
5095	NOREF(pCtx);
5096
5097	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5098	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5099	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5100	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5101
5102	#ifdef VBOX_STRICT
5103	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5104	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5105
5106	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5107	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5108	#endif
5109
5110	/* Disable interrupts. */
5111	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5112
5113	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5114	RTCPUID idHostCpu = RTMpCpuId();
5115	CPUMR0SetLApic(pVCpu, idHostCpu);
5116	#endif
5117
5118	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5119	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5120
5121	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5122	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5123
5124	/* Leave VMX Root Mode. */
5125	VMXDisable();
5126
5127	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5128
5129	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5130	CPUMSetHyperEIP(pVCpu, enmOp);
5131	for (int i = (int)cParams - 1; i >= 0; i--)
5132	CPUMPushHyper(pVCpu, paParam[i]);
5133
5134	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5135
5136	/* Call the switcher. */
5137	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5138	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5139
5140	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5141	/* Make sure the VMX instructions don't cause #UD faults. */
5142	SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5143
5144	/* Re-enter VMX Root Mode */
5145	int rc2 = VMXEnable(HCPhysCpuPage);
5146	if (RT_FAILURE(rc2))
5147	{
5148	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5149	ASMSetFlags(fOldEFlags);
5150	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5151	return rc2;
5152	}
5153
5154	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5155	AssertRC(rc2);
5156	Assert(!(ASMGetFlags() & X86_EFL_IF));
5157	ASMSetFlags(fOldEFlags);
5158	return rc;
5159	}
5160
5161
5162	/**
5163	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5164	* supporting 64-bit guests.
5165	*
5166	* @returns VBox status code.
5167	* @param fResume Whether to VMLAUNCH or VMRESUME.
5168	* @param pCtx Pointer to the guest-CPU context.
5169	* @param pCache Pointer to the VMCS cache.
5170	* @param pVM The cross context VM structure.
5171	* @param pVCpu The cross context virtual CPU structure.
5172	*/
5173	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5174	{
5175	NOREF(fResume);
5176
5177	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5178	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5179
5180	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5181	pCache->uPos = 1;
5182	pCache->interPD = PGMGetInterPaeCR3(pVM);
5183	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5184	#endif
5185
5186	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5187	pCache->TestIn.HCPhysCpuPage = 0;
5188	pCache->TestIn.HCPhysVmcs = 0;
5189	pCache->TestIn.pCache = 0;
5190	pCache->TestOut.HCPhysVmcs = 0;
5191	pCache->TestOut.pCache = 0;
5192	pCache->TestOut.pCtx = 0;
5193	pCache->TestOut.eflags = 0;
5194	#else
5195	NOREF(pCache);
5196	#endif
5197
5198	uint32_t aParam[10];
5199	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5200	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5201	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5202	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5203	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5204	aParam[5] = 0;
5205	aParam[6] = VM_RC_ADDR(pVM, pVM);
5206	aParam[7] = 0;
5207	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5208	aParam[9] = 0;
5209
5210	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5211	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5212	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5213	#endif
5214	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5215
5216	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5217	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5218	Assert(pCtx->dr[4] == 10);
5219	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5220	#endif
5221
5222	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5223	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5224	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5225	pVCpu->hm.s.vmx.HCPhysVmcs));
5226	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5227	pCache->TestOut.HCPhysVmcs));
5228	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5229	pCache->TestOut.pCache));
5230	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5231	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5232	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5233	pCache->TestOut.pCtx));
5234	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5235	#endif
5236	return rc;
5237	}
5238
5239
5240	/**
5241	* Initialize the VMCS-Read cache.
5242	*
5243	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5244	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5245	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5246	* (those that have a 32-bit FULL & HIGH part).
5247	*
5248	* @returns VBox status code.
5249	* @param pVM The cross context VM structure.
5250	* @param pVCpu The cross context virtual CPU structure.
5251	*/
5252	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5253	{
5254	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5255	{ \
5256	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5257	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5258	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5259	++cReadFields; \
5260	}
5261
5262	AssertPtr(pVM);
5263	AssertPtr(pVCpu);
5264	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5265	uint32_t cReadFields = 0;
5266
5267	/*
5268	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5269	* and serve to indicate exceptions to the rules.
5270	*/
5271
5272	/* Guest-natural selector base fields. */
5273	#if 0
5274	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5277	#endif
5278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5290	#if 0
5291	/* Unused natural width guest-state fields. */
5292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5294	#endif
5295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5297
5298	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5299	#if 0
5300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5309	#endif
5310
5311	/* Natural width guest-state fields. */
5312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5313	#if 0
5314	/* Currently unused field. */
5315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5316	#endif
5317
5318	if (pVM->hm.s.fNestedPaging)
5319	{
5320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5321	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5322	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5323	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5324	}
5325	else
5326	{
5327	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5328	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5329	}
5330
5331	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5332	return VINF_SUCCESS;
5333	}
5334
5335
5336	/**
5337	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5338	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5339	* darwin, running 64-bit guests).
5340	*
5341	* @returns VBox status code.
5342	* @param pVCpu The cross context virtual CPU structure.
5343	* @param idxField The VMCS field encoding.
5344	* @param u64Val 16, 32 or 64-bit value.
5345	*/
5346	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5347	{
5348	int rc;
5349	switch (idxField)
5350	{
5351	/*
5352	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5353	*/
5354	/* 64-bit Control fields. */
5355	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5356	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5357	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5358	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5359	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5360	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5361	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5362	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5363	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5364	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5365	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5366	case VMX_VMCS64_CTRL_EPTP_FULL:
5367	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5368	/* 64-bit Guest-state fields. */
5369	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5370	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5371	case VMX_VMCS64_GUEST_PAT_FULL:
5372	case VMX_VMCS64_GUEST_EFER_FULL:
5373	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5374	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5375	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5376	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5377	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5378	/* 64-bit Host-state fields. */
5379	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5380	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5381	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5382	{
5383	rc = VMXWriteVmcs32(idxField, u64Val);
5384	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5385	break;
5386	}
5387
5388	/*
5389	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5390	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5391	*/
5392	/* Natural-width Guest-state fields. */
5393	case VMX_VMCS_GUEST_CR3:
5394	case VMX_VMCS_GUEST_ES_BASE:
5395	case VMX_VMCS_GUEST_CS_BASE:
5396	case VMX_VMCS_GUEST_SS_BASE:
5397	case VMX_VMCS_GUEST_DS_BASE:
5398	case VMX_VMCS_GUEST_FS_BASE:
5399	case VMX_VMCS_GUEST_GS_BASE:
5400	case VMX_VMCS_GUEST_LDTR_BASE:
5401	case VMX_VMCS_GUEST_TR_BASE:
5402	case VMX_VMCS_GUEST_GDTR_BASE:
5403	case VMX_VMCS_GUEST_IDTR_BASE:
5404	case VMX_VMCS_GUEST_RSP:
5405	case VMX_VMCS_GUEST_RIP:
5406	case VMX_VMCS_GUEST_SYSENTER_ESP:
5407	case VMX_VMCS_GUEST_SYSENTER_EIP:
5408	{
5409	if (!(u64Val >> 32))
5410	{
5411	/* If this field is 64-bit, VT-x will zero out the top bits. */
5412	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5413	}
5414	else
5415	{
5416	/* Assert that only the 32->64 switcher case should ever come here. */
5417	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5418	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5419	}
5420	break;
5421	}
5422
5423	default:
5424	{
5425	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5426	rc = VERR_INVALID_PARAMETER;
5427	break;
5428	}
5429	}
5430	AssertRCReturn(rc, rc);
5431	return rc;
5432	}
5433
5434
5435	/**
5436	* Queue up a VMWRITE by using the VMCS write cache.
5437	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5438	*
5439	* @param pVCpu The cross context virtual CPU structure.
5440	* @param idxField The VMCS field encoding.
5441	* @param u64Val 16, 32 or 64-bit value.
5442	*/
5443	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5444	{
5445	AssertPtr(pVCpu);
5446	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5447
5448	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5449	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5450
5451	/* Make sure there are no duplicates. */
5452	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5453	{
5454	if (pCache->Write.aField[i] == idxField)
5455	{
5456	pCache->Write.aFieldVal[i] = u64Val;
5457	return VINF_SUCCESS;
5458	}
5459	}
5460
5461	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5462	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5463	pCache->Write.cValidEntries++;
5464	return VINF_SUCCESS;
5465	}
5466	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5467
5468
5469	/**
5470	* Sets up the usage of TSC-offsetting and updates the VMCS.
5471	*
5472	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5473	* VMX preemption timer.
5474	*
5475	* @returns VBox status code.
5476	* @param pVM The cross context VM structure.
5477	* @param pVCpu The cross context virtual CPU structure.
5478	*
5479	* @remarks No-long-jump zone!!!
5480	*/
5481	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5482	{
5483	int rc;
5484	bool fOffsettedTsc;
5485	bool fParavirtTsc;
5486	if (pVM->hm.s.vmx.fUsePreemptTimer)
5487	{
5488	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5489	&fOffsettedTsc, &fParavirtTsc);
5490
5491	/* Make sure the returned values have sane upper and lower boundaries. */
5492	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5493	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5494	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5495	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5496
5497	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5498	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5499	}
5500	else
5501	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5502
5503	/** @todo later optimize this to be done elsewhere and not before every
5504	* VM-entry. */
5505	if (fParavirtTsc)
5506	{
5507	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5508	information before every VM-entry, hence disable it for performance sake. */
5509	#if 0
5510	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5511	AssertRC(rc);
5512	#endif
5513	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5514	}
5515
5516	if (fOffsettedTsc)
5517	{
5518	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5519	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5520
5521	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5522	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5523	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5524	}
5525	else
5526	{
5527	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5528	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5529	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5530	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5531	}
5532	}
5533
5534
5535	/**
5536	* Determines if an exception is a contributory exception.
5537	*
5538	* Contributory exceptions are ones which can cause double-faults unless the
5539	* original exception was a benign exception. Page-fault is intentionally not
5540	* included here as it's a conditional contributory exception.
5541	*
5542	* @returns true if the exception is contributory, false otherwise.
5543	* @param uVector The exception vector.
5544	*/
5545	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5546	{
5547	switch (uVector)
5548	{
5549	case X86_XCPT_GP:
5550	case X86_XCPT_SS:
5551	case X86_XCPT_NP:
5552	case X86_XCPT_TS:
5553	case X86_XCPT_DE:
5554	return true;
5555	default:
5556	break;
5557	}
5558	return false;
5559	}
5560
5561
5562	/**
5563	* Sets an event as a pending event to be injected into the guest.
5564	*
5565	* @param pVCpu The cross context virtual CPU structure.
5566	* @param u32IntInfo The VM-entry interruption-information field.
5567	* @param cbInstr The VM-entry instruction length in bytes (for software
5568	* interrupts, exceptions and privileged software
5569	* exceptions).
5570	* @param u32ErrCode The VM-entry exception error code.
5571	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5572	* page-fault.
5573	*
5574	* @remarks Statistics counter assumes this is a guest event being injected or
5575	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5576	* always incremented.
5577	*/
5578	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5579	RTGCUINTPTR GCPtrFaultAddress)
5580	{
5581	Assert(!pVCpu->hm.s.Event.fPending);
5582	pVCpu->hm.s.Event.fPending = true;
5583	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5584	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5585	pVCpu->hm.s.Event.cbInstr = cbInstr;
5586	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5587
5588	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5589	}
5590
5591
5592	/**
5593	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5594	*
5595	* @param pVCpu The cross context virtual CPU structure.
5596	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5597	* out-of-sync. Make sure to update the required fields
5598	* before using them.
5599	*/
5600	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5601	{
5602	NOREF(pMixedCtx);
5603	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5604	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5605	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5606	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5607	}
5608
5609
5610	/**
5611	* Handle a condition that occurred while delivering an event through the guest
5612	* IDT.
5613	*
5614	* @returns Strict VBox status code (informational error codes included).
5615	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5616	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5617	* to continue execution of the guest which will delivery the \#DF.
5618	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5619	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5620	*
5621	* @param pVCpu The cross context virtual CPU structure.
5622	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5623	* out-of-sync. Make sure to update the required fields
5624	* before using them.
5625	* @param pVmxTransient Pointer to the VMX transient structure.
5626	*
5627	* @remarks No-long-jump zone!!!
5628	*/
5629	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5630	{
5631	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5632
5633	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5634	AssertRCReturn(rc2, rc2);
5635	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5636	AssertRCReturn(rc2, rc2);
5637
5638	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5639	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5640	{
5641	uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5642	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5643
5644	typedef enum
5645	{
5646	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5647	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5648	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5649	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5650	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5651	} VMXREFLECTXCPT;
5652
5653	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5654	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5655	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5656	{
5657	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5658	{
5659	enmReflect = VMXREFLECTXCPT_XCPT;
5660	#ifdef VBOX_STRICT
5661	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5662	&& uExitVector == X86_XCPT_PF)
5663	{
5664	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5665	}
5666	#endif
5667	if ( uExitVector == X86_XCPT_PF
5668	&& uIdtVector == X86_XCPT_PF)
5669	{
5670	pVmxTransient->fVectoringDoublePF = true;
5671	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5672	}
5673	else if ( uExitVector == X86_XCPT_AC
5674	&& uIdtVector == X86_XCPT_AC)
5675	{
5676	enmReflect = VMXREFLECTXCPT_HANG;
5677	Log4(("IDT: Nested #AC - Bad guest\n"));
5678	}
5679	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5680	&& hmR0VmxIsContributoryXcpt(uExitVector)
5681	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5682	\|\| uIdtVector == X86_XCPT_PF))
5683	{
5684	enmReflect = VMXREFLECTXCPT_DF;
5685	}
5686	else if (uIdtVector == X86_XCPT_DF)
5687	enmReflect = VMXREFLECTXCPT_TF;
5688	}
5689	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5690	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5691	{
5692	/*
5693	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5694	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5695	*/
5696	enmReflect = VMXREFLECTXCPT_XCPT;
5697
5698	if (uExitVector == X86_XCPT_PF)
5699	{
5700	pVmxTransient->fVectoringPF = true;
5701	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5702	}
5703	}
5704	}
5705	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5706	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5707	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5708	{
5709	/*
5710	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5711	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5712	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5713	*/
5714	enmReflect = VMXREFLECTXCPT_XCPT;
5715	}
5716
5717	/*
5718	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5719	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5720	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5721	*
5722	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5723	*/
5724	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5725	&& enmReflect == VMXREFLECTXCPT_XCPT
5726	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5727	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5728	{
5729	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5730	}
5731
5732	switch (enmReflect)
5733	{
5734	case VMXREFLECTXCPT_XCPT:
5735	{
5736	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5737	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5738	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5739
5740	uint32_t u32ErrCode = 0;
5741	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5742	{
5743	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5744	AssertRCReturn(rc2, rc2);
5745	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5746	}
5747
5748	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5749	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5750	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5751	rcStrict = VINF_SUCCESS;
5752	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5753	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5754
5755	break;
5756	}
5757
5758	case VMXREFLECTXCPT_DF:
5759	{
5760	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5761	rcStrict = VINF_HM_DOUBLE_FAULT;
5762	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5763	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5764
5765	break;
5766	}
5767
5768	case VMXREFLECTXCPT_TF:
5769	{
5770	rcStrict = VINF_EM_RESET;
5771	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5772	uExitVector));
5773	break;
5774	}
5775
5776	case VMXREFLECTXCPT_HANG:
5777	{
5778	rcStrict = VERR_EM_GUEST_CPU_HANG;
5779	break;
5780	}
5781
5782	default:
5783	Assert(rcStrict == VINF_SUCCESS);
5784	break;
5785	}
5786	}
5787	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5788	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5789	&& uExitVector != X86_XCPT_DF
5790	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5791	{
5792	/*
5793	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5794	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5795	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5796	*/
5797	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5798	{
5799	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5800	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5801	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5802	}
5803	}
5804
5805	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
5806	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
5807	return rcStrict;
5808	}
5809
5810
5811	/**
5812	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5813	*
5814	* @returns VBox status code.
5815	* @param pVCpu The cross context virtual CPU structure.
5816	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5817	* out-of-sync. Make sure to update the required fields
5818	* before using them.
5819	*
5820	* @remarks No-long-jump zone!!!
5821	*/
5822	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5823	{
5824	NOREF(pMixedCtx);
5825
5826	/*
5827	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5828	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5829	*/
5830	VMMRZCallRing3Disable(pVCpu);
5831	HM_DISABLE_PREEMPT();
5832
5833	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5834	{
5835	uint32_t uVal = 0;
5836	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5837	AssertRCReturn(rc, rc);
5838
5839	uint32_t uShadow = 0;
5840	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5841	AssertRCReturn(rc, rc);
5842
5843	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5844	CPUMSetGuestCR0(pVCpu, uVal);
5845	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5846	}
5847
5848	HM_RESTORE_PREEMPT();
5849	VMMRZCallRing3Enable(pVCpu);
5850	return VINF_SUCCESS;
5851	}
5852
5853
5854	/**
5855	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5856	*
5857	* @returns VBox status code.
5858	* @param pVCpu The cross context virtual CPU structure.
5859	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5860	* out-of-sync. Make sure to update the required fields
5861	* before using them.
5862	*
5863	* @remarks No-long-jump zone!!!
5864	*/
5865	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5866	{
5867	NOREF(pMixedCtx);
5868
5869	int rc = VINF_SUCCESS;
5870	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5871	{
5872	uint32_t uVal = 0;
5873	uint32_t uShadow = 0;
5874	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5875	AssertRCReturn(rc, rc);
5876	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5877	AssertRCReturn(rc, rc);
5878
5879	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5880	CPUMSetGuestCR4(pVCpu, uVal);
5881	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5882	}
5883	return rc;
5884	}
5885
5886
5887	/**
5888	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5889	*
5890	* @returns VBox status code.
5891	* @param pVCpu The cross context virtual CPU structure.
5892	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5893	* out-of-sync. Make sure to update the required fields
5894	* before using them.
5895	*
5896	* @remarks No-long-jump zone!!!
5897	*/
5898	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5899	{
5900	int rc = VINF_SUCCESS;
5901	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5902	{
5903	uint64_t u64Val = 0;
5904	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5905	AssertRCReturn(rc, rc);
5906
5907	pMixedCtx->rip = u64Val;
5908	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5909	}
5910	return rc;
5911	}
5912
5913
5914	/**
5915	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5916	*
5917	* @returns VBox status code.
5918	* @param pVCpu The cross context virtual CPU structure.
5919	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5920	* out-of-sync. Make sure to update the required fields
5921	* before using them.
5922	*
5923	* @remarks No-long-jump zone!!!
5924	*/
5925	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5926	{
5927	int rc = VINF_SUCCESS;
5928	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5929	{
5930	uint64_t u64Val = 0;
5931	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5932	AssertRCReturn(rc, rc);
5933
5934	pMixedCtx->rsp = u64Val;
5935	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5936	}
5937	return rc;
5938	}
5939
5940
5941	/**
5942	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5943	*
5944	* @returns VBox status code.
5945	* @param pVCpu The cross context virtual CPU structure.
5946	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5947	* out-of-sync. Make sure to update the required fields
5948	* before using them.
5949	*
5950	* @remarks No-long-jump zone!!!
5951	*/
5952	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5953	{
5954	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5955	{
5956	uint32_t uVal = 0;
5957	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5958	AssertRCReturn(rc, rc);
5959
5960	pMixedCtx->eflags.u32 = uVal;
5961	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5962	{
5963	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5964	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5965
5966	pMixedCtx->eflags.Bits.u1VM = 0;
5967	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5968	}
5969
5970	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5971	}
5972	return VINF_SUCCESS;
5973	}
5974
5975
5976	/**
5977	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5978	* guest-CPU context.
5979	*/
5980	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5981	{
5982	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5983	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5984	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5985	return rc;
5986	}
5987
5988
5989	/**
5990	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5991	* from the guest-state area in the VMCS.
5992	*
5993	* @param pVCpu The cross context virtual CPU structure.
5994	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5995	* out-of-sync. Make sure to update the required fields
5996	* before using them.
5997	*
5998	* @remarks No-long-jump zone!!!
5999	*/
6000	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6001	{
6002	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6003	{
6004	uint32_t uIntrState = 0;
6005	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6006	AssertRC(rc);
6007
6008	if (!uIntrState)
6009	{
6010	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6011	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6012
6013	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6014	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6015	}
6016	else
6017	{
6018	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6019	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6020	{
6021	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6022	AssertRC(rc);
6023	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6024	AssertRC(rc);
6025
6026	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6027	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6028	}
6029	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6030	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6031
6032	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6033	{
6034	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6035	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6036	}
6037	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6038	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6039	}
6040
6041	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6042	}
6043	}
6044
6045
6046	/**
6047	* Saves the guest's activity state.
6048	*
6049	* @returns VBox status code.
6050	* @param pVCpu The cross context virtual CPU structure.
6051	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6052	* out-of-sync. Make sure to update the required fields
6053	* before using them.
6054	*
6055	* @remarks No-long-jump zone!!!
6056	*/
6057	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6058	{
6059	NOREF(pMixedCtx);
6060	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6061	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6062	return VINF_SUCCESS;
6063	}
6064
6065
6066	/**
6067	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6068	* the current VMCS into the guest-CPU context.
6069	*
6070	* @returns VBox status code.
6071	* @param pVCpu The cross context virtual CPU structure.
6072	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6073	* out-of-sync. Make sure to update the required fields
6074	* before using them.
6075	*
6076	* @remarks No-long-jump zone!!!
6077	*/
6078	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6079	{
6080	int rc = VINF_SUCCESS;
6081	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6082	{
6083	uint32_t u32Val = 0;
6084	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6085	pMixedCtx->SysEnter.cs = u32Val;
6086	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6087	}
6088
6089	uint64_t u64Val = 0;
6090	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6091	{
6092	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6093	pMixedCtx->SysEnter.eip = u64Val;
6094	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6095	}
6096	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6097	{
6098	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6099	pMixedCtx->SysEnter.esp = u64Val;
6100	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6101	}
6102	return rc;
6103	}
6104
6105
6106	/**
6107	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6108	* the CPU back into the guest-CPU context.
6109	*
6110	* @returns VBox status code.
6111	* @param pVCpu The cross context virtual CPU structure.
6112	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6113	* out-of-sync. Make sure to update the required fields
6114	* before using them.
6115	*
6116	* @remarks No-long-jump zone!!!
6117	*/
6118	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6119	{
6120	#if HC_ARCH_BITS == 64
6121	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6122	{
6123	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6124	VMMRZCallRing3Disable(pVCpu);
6125	HM_DISABLE_PREEMPT();
6126
6127	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6128	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6129	{
6130	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6131	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6132	}
6133
6134	HM_RESTORE_PREEMPT();
6135	VMMRZCallRing3Enable(pVCpu);
6136	}
6137	else
6138	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6139	#else
6140	NOREF(pMixedCtx);
6141	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6142	#endif
6143
6144	return VINF_SUCCESS;
6145	}
6146
6147
6148	/**
6149	* Saves the auto load/store'd guest MSRs from the current VMCS into
6150	* the guest-CPU context.
6151	*
6152	* @returns VBox status code.
6153	* @param pVCpu The cross context virtual CPU structure.
6154	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6155	* out-of-sync. Make sure to update the required fields
6156	* before using them.
6157	*
6158	* @remarks No-long-jump zone!!!
6159	*/
6160	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6161	{
6162	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6163	return VINF_SUCCESS;
6164
6165	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6166	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6167	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6168	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6169	{
6170	switch (pMsr->u32Msr)
6171	{
6172	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6173	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6174	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6175	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6176	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6177	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6178	break;
6179
6180	default:
6181	{
6182	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6183	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6184	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6185	}
6186	}
6187	}
6188
6189	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6190	return VINF_SUCCESS;
6191	}
6192
6193
6194	/**
6195	* Saves the guest control registers from the current VMCS into the guest-CPU
6196	* context.
6197	*
6198	* @returns VBox status code.
6199	* @param pVCpu The cross context virtual CPU structure.
6200	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6201	* out-of-sync. Make sure to update the required fields
6202	* before using them.
6203	*
6204	* @remarks No-long-jump zone!!!
6205	*/
6206	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6207	{
6208	/* Guest CR0. Guest FPU. */
6209	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6210	AssertRCReturn(rc, rc);
6211
6212	/* Guest CR4. */
6213	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6214	AssertRCReturn(rc, rc);
6215
6216	/* Guest CR2 - updated always during the world-switch or in #PF. */
6217	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6218	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6219	{
6220	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6221	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6222
6223	PVM pVM = pVCpu->CTX_SUFF(pVM);
6224	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6225	\|\| ( pVM->hm.s.fNestedPaging
6226	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6227	{
6228	uint64_t u64Val = 0;
6229	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6230	if (pMixedCtx->cr3 != u64Val)
6231	{
6232	CPUMSetGuestCR3(pVCpu, u64Val);
6233	if (VMMRZCallRing3IsEnabled(pVCpu))
6234	{
6235	PGMUpdateCR3(pVCpu, u64Val);
6236	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6237	}
6238	else
6239	{
6240	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6241	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6242	}
6243	}
6244
6245	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6246	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6247	{
6248	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6249	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6250	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6251	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6252
6253	if (VMMRZCallRing3IsEnabled(pVCpu))
6254	{
6255	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6256	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6257	}
6258	else
6259	{
6260	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6261	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6262	}
6263	}
6264	}
6265
6266	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6267	}
6268
6269	/*
6270	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6271	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6272	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6273	*
6274	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6275	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6276	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6277	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6278	*
6279	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6280	*/
6281	if (VMMRZCallRing3IsEnabled(pVCpu))
6282	{
6283	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6284	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6285
6286	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6287	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6288
6289	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6290	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6291	}
6292
6293	return rc;
6294	}
6295
6296
6297	/**
6298	* Reads a guest segment register from the current VMCS into the guest-CPU
6299	* context.
6300	*
6301	* @returns VBox status code.
6302	* @param pVCpu The cross context virtual CPU structure.
6303	* @param idxSel Index of the selector in the VMCS.
6304	* @param idxLimit Index of the segment limit in the VMCS.
6305	* @param idxBase Index of the segment base in the VMCS.
6306	* @param idxAccess Index of the access rights of the segment in the VMCS.
6307	* @param pSelReg Pointer to the segment selector.
6308	*
6309	* @remarks No-long-jump zone!!!
6310	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6311	* macro as that takes care of whether to read from the VMCS cache or
6312	* not.
6313	*/
6314	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6315	PCPUMSELREG pSelReg)
6316	{
6317	NOREF(pVCpu);
6318
6319	uint32_t u32Val = 0;
6320	int rc = VMXReadVmcs32(idxSel, &u32Val);
6321	AssertRCReturn(rc, rc);
6322	pSelReg->Sel = (uint16_t)u32Val;
6323	pSelReg->ValidSel = (uint16_t)u32Val;
6324	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6325
6326	rc = VMXReadVmcs32(idxLimit, &u32Val);
6327	AssertRCReturn(rc, rc);
6328	pSelReg->u32Limit = u32Val;
6329
6330	uint64_t u64Val = 0;
6331	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6332	AssertRCReturn(rc, rc);
6333	pSelReg->u64Base = u64Val;
6334
6335	rc = VMXReadVmcs32(idxAccess, &u32Val);
6336	AssertRCReturn(rc, rc);
6337	pSelReg->Attr.u = u32Val;
6338
6339	/*
6340	* If VT-x marks the segment as unusable, most other bits remain undefined:
6341	* - For CS the L, D and G bits have meaning.
6342	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6343	* - For the remaining data segments no bits are defined.
6344	*
6345	* The present bit and the unusable bit has been observed to be set at the
6346	* same time (the selector was supposed to be invalid as we started executing
6347	* a V8086 interrupt in ring-0).
6348	*
6349	* What should be important for the rest of the VBox code, is that the P bit is
6350	* cleared. Some of the other VBox code recognizes the unusable bit, but
6351	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6352	* safe side here, we'll strip off P and other bits we don't care about. If
6353	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6354	*
6355	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6356	*/
6357	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6358	{
6359	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6360
6361	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6362	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6363	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6364
6365	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6366	#ifdef DEBUG_bird
6367	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6368	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6369	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6370	#endif
6371	}
6372	return VINF_SUCCESS;
6373	}
6374
6375
6376	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6377	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6378	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6379	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6380	#else
6381	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6382	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6383	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6384	#endif
6385
6386
6387	/**
6388	* Saves the guest segment registers from the current VMCS into the guest-CPU
6389	* context.
6390	*
6391	* @returns VBox status code.
6392	* @param pVCpu The cross context virtual CPU structure.
6393	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6394	* out-of-sync. Make sure to update the required fields
6395	* before using them.
6396	*
6397	* @remarks No-long-jump zone!!!
6398	*/
6399	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6400	{
6401	/* Guest segment registers. */
6402	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6403	{
6404	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6405	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6406	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6407	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6408	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6409	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6410	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6411
6412	/* Restore segment attributes for real-on-v86 mode hack. */
6413	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6414	{
6415	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6416	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6417	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6418	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6419	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6420	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6421	}
6422	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6423	}
6424
6425	return VINF_SUCCESS;
6426	}
6427
6428
6429	/**
6430	* Saves the guest descriptor table registers and task register from the current
6431	* VMCS into the guest-CPU context.
6432	*
6433	* @returns VBox status code.
6434	* @param pVCpu The cross context virtual CPU structure.
6435	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6436	* out-of-sync. Make sure to update the required fields
6437	* before using them.
6438	*
6439	* @remarks No-long-jump zone!!!
6440	*/
6441	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6442	{
6443	int rc = VINF_SUCCESS;
6444
6445	/* Guest LDTR. */
6446	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6447	{
6448	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6449	AssertRCReturn(rc, rc);
6450	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6451	}
6452
6453	/* Guest GDTR. */
6454	uint64_t u64Val = 0;
6455	uint32_t u32Val = 0;
6456	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6457	{
6458	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6459	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6460	pMixedCtx->gdtr.pGdt = u64Val;
6461	pMixedCtx->gdtr.cbGdt = u32Val;
6462	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6463	}
6464
6465	/* Guest IDTR. */
6466	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6467	{
6468	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6469	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6470	pMixedCtx->idtr.pIdt = u64Val;
6471	pMixedCtx->idtr.cbIdt = u32Val;
6472	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6473	}
6474
6475	/* Guest TR. */
6476	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6477	{
6478	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6479	AssertRCReturn(rc, rc);
6480
6481	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6482	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6483	{
6484	rc = VMXLOCAL_READ_SEG(TR, tr);
6485	AssertRCReturn(rc, rc);
6486	}
6487	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6488	}
6489	return rc;
6490	}
6491
6492	#undef VMXLOCAL_READ_SEG
6493
6494
6495	/**
6496	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6497	* context.
6498	*
6499	* @returns VBox status code.
6500	* @param pVCpu The cross context virtual CPU structure.
6501	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6502	* out-of-sync. Make sure to update the required fields
6503	* before using them.
6504	*
6505	* @remarks No-long-jump zone!!!
6506	*/
6507	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6508	{
6509	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6510	{
6511	if (!pVCpu->hm.s.fUsingHyperDR7)
6512	{
6513	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6514	uint32_t u32Val;
6515	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6516	pMixedCtx->dr[7] = u32Val;
6517	}
6518
6519	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6520	}
6521	return VINF_SUCCESS;
6522	}
6523
6524
6525	/**
6526	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6527	*
6528	* @returns VBox status code.
6529	* @param pVCpu The cross context virtual CPU structure.
6530	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6531	* out-of-sync. Make sure to update the required fields
6532	* before using them.
6533	*
6534	* @remarks No-long-jump zone!!!
6535	*/
6536	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6537	{
6538	NOREF(pMixedCtx);
6539
6540	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6541	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6542	return VINF_SUCCESS;
6543	}
6544
6545
6546	/**
6547	* Saves the entire guest state from the currently active VMCS into the
6548	* guest-CPU context.
6549	*
6550	* This essentially VMREADs all guest-data.
6551	*
6552	* @returns VBox status code.
6553	* @param pVCpu The cross context virtual CPU structure.
6554	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6555	* out-of-sync. Make sure to update the required fields
6556	* before using them.
6557	*/
6558	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6559	{
6560	Assert(pVCpu);
6561	Assert(pMixedCtx);
6562
6563	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6564	return VINF_SUCCESS;
6565
6566	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6567	again on the ring-3 callback path, there is no real need to. */
6568	if (VMMRZCallRing3IsEnabled(pVCpu))
6569	VMMR0LogFlushDisable(pVCpu);
6570	else
6571	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6572	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6573
6574	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6575	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6576
6577	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6578	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6579
6580	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6581	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6582
6583	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6584	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6585
6586	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6587	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6588
6589	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6590	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6591
6592	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6593	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6594
6595	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6596	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6597
6598	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6599	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6600
6601	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6602	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6603
6604	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6605	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6606
6607	if (VMMRZCallRing3IsEnabled(pVCpu))
6608	VMMR0LogFlushEnable(pVCpu);
6609
6610	return VINF_SUCCESS;
6611	}
6612
6613
6614	/**
6615	* Saves basic guest registers needed for IEM instruction execution.
6616	*
6617	* @returns VBox status code (OR-able).
6618	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
6619	* @param pMixedCtx Pointer to the CPU context of the guest.
6620	* @param fMemory Whether the instruction being executed operates on
6621	* memory or not. Only CR0 is synced up if clear.
6622	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6623	*/
6624	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6625	{
6626	/*
6627	* We assume all general purpose registers other than RSP are available.
6628	*
6629	* RIP is a must, as it will be incremented or otherwise changed.
6630	*
6631	* RFLAGS are always required to figure the CPL.
6632	*
6633	* RSP isn't always required, however it's a GPR, so frequently required.
6634	*
6635	* SS and CS are the only segment register needed if IEM doesn't do memory
6636	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6637	*
6638	* CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6639	* be required for memory accesses.
6640	*
6641	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
6642	*/
6643	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6644	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6645	if (fNeedRsp)
6646	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6647	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6648	if (!fMemory)
6649	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6650	else
6651	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6652	AssertRCReturn(rc, rc);
6653	return rc;
6654	}
6655
6656
6657	/**
6658	* Ensures that we've got a complete basic guest-context.
6659	*
6660	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
6661	* is for the interpreter.
6662	*
6663	* @returns VBox status code.
6664	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
6665	* @param pMixedCtx Pointer to the guest-CPU context which may have data
6666	* needing to be synced in.
6667	* @thread EMT(pVCpu)
6668	*/
6669	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6670	{
6671	/* Note! Since this is only applicable to VT-x, the implementation is placed
6672	in the VT-x part of the sources instead of the generic stuff. */
6673	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6674	return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6675	return VINF_SUCCESS;
6676	}
6677
6678
6679	/**
6680	* Check per-VM and per-VCPU force flag actions that require us to go back to
6681	* ring-3 for one reason or another.
6682	*
6683	* @returns Strict VBox status code (information status code included).
6684	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6685	* ring-3.
6686	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6687	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6688	* interrupts)
6689	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6690	* all EMTs to be in ring-3.
6691	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6692	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6693	* to the EM loop.
6694	*
6695	* @param pVM The cross context VM structure.
6696	* @param pVCpu The cross context virtual CPU structure.
6697	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6698	* out-of-sync. Make sure to update the required fields
6699	* before using them.
6700	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
6701	*/
6702	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6703	{
6704	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6705
6706	/*
6707	* Anything pending? Should be more likely than not if we're doing a good job.
6708	*/
6709	if ( !fStepping
6710	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6711	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6712	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6713	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6714	return VINF_SUCCESS;
6715
6716	/* We need the control registers now, make sure the guest-CPU context is updated. */
6717	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6718	AssertRCReturn(rc3, rc3);
6719
6720	/* Pending HM CR3 sync. */
6721	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6722	{
6723	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6724	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6725	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6726	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6727	}
6728
6729	/* Pending HM PAE PDPEs. */
6730	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6731	{
6732	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6733	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6734	}
6735
6736	/* Pending PGM C3 sync. */
6737	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6738	{
6739	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6740	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6741	if (rcStrict2 != VINF_SUCCESS)
6742	{
6743	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6744	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6745	return rcStrict2;
6746	}
6747	}
6748
6749	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6750	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6751	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6752	{
6753	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6754	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6755	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6756	return rc2;
6757	}
6758
6759	/* Pending VM request packets, such as hardware interrupts. */
6760	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6761	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6762	{
6763	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6764	return VINF_EM_PENDING_REQUEST;
6765	}
6766
6767	/* Pending PGM pool flushes. */
6768	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6769	{
6770	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6771	return VINF_PGM_POOL_FLUSH_PENDING;
6772	}
6773
6774	/* Pending DMA requests. */
6775	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6776	{
6777	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6778	return VINF_EM_RAW_TO_R3;
6779	}
6780
6781	return VINF_SUCCESS;
6782	}
6783
6784
6785	/**
6786	* Converts any TRPM trap into a pending HM event. This is typically used when
6787	* entering from ring-3 (not longjmp returns).
6788	*
6789	* @param pVCpu The cross context virtual CPU structure.
6790	*/
6791	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6792	{
6793	Assert(TRPMHasTrap(pVCpu));
6794	Assert(!pVCpu->hm.s.Event.fPending);
6795
6796	uint8_t uVector;
6797	TRPMEVENT enmTrpmEvent;
6798	RTGCUINT uErrCode;
6799	RTGCUINTPTR GCPtrFaultAddress;
6800	uint8_t cbInstr;
6801
6802	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6803	AssertRC(rc);
6804
6805	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6806	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6807	if (enmTrpmEvent == TRPM_TRAP)
6808	{
6809	switch (uVector)
6810	{
6811	case X86_XCPT_NMI:
6812	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6813	break;
6814
6815	case X86_XCPT_BP:
6816	case X86_XCPT_OF:
6817	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6818	break;
6819
6820	case X86_XCPT_PF:
6821	case X86_XCPT_DF:
6822	case X86_XCPT_TS:
6823	case X86_XCPT_NP:
6824	case X86_XCPT_SS:
6825	case X86_XCPT_GP:
6826	case X86_XCPT_AC:
6827	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6828	/* no break! */
6829	default:
6830	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6831	break;
6832	}
6833	}
6834	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6835	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6836	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6837	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6838	else
6839	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6840
6841	rc = TRPMResetTrap(pVCpu);
6842	AssertRC(rc);
6843	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6844	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6845
6846	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6847	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6848	}
6849
6850
6851	/**
6852	* Converts the pending HM event into a TRPM trap.
6853	*
6854	* @param pVCpu The cross context virtual CPU structure.
6855	*/
6856	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6857	{
6858	Assert(pVCpu->hm.s.Event.fPending);
6859
6860	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6861	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6862	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6863	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6864
6865	/* If a trap was already pending, we did something wrong! */
6866	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6867
6868	TRPMEVENT enmTrapType;
6869	switch (uVectorType)
6870	{
6871	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6872	enmTrapType = TRPM_HARDWARE_INT;
6873	break;
6874
6875	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6876	enmTrapType = TRPM_SOFTWARE_INT;
6877	break;
6878
6879	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6880	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6881	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6882	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6883	enmTrapType = TRPM_TRAP;
6884	break;
6885
6886	default:
6887	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6888	enmTrapType = TRPM_32BIT_HACK;
6889	break;
6890	}
6891
6892	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6893
6894	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6895	AssertRC(rc);
6896
6897	if (fErrorCodeValid)
6898	TRPMSetErrorCode(pVCpu, uErrorCode);
6899
6900	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6901	&& uVector == X86_XCPT_PF)
6902	{
6903	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6904	}
6905	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6906	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6907	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6908	{
6909	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6910	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6911	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6912	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6913	}
6914
6915	/* Clear any pending events from the VMCS. */
6916	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6917	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6918
6919	/* We're now done converting the pending event. */
6920	pVCpu->hm.s.Event.fPending = false;
6921	}
6922
6923
6924	/**
6925	* Does the necessary state syncing before returning to ring-3 for any reason
6926	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6927	*
6928	* @returns VBox status code.
6929	* @param pVM The cross context VM structure.
6930	* @param pVCpu The cross context virtual CPU structure.
6931	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6932	* be out-of-sync. Make sure to update the required
6933	* fields before using them.
6934	* @param fSaveGuestState Whether to save the guest state or not.
6935	*
6936	* @remarks No-long-jmp zone!!!
6937	*/
6938	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6939	{
6940	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6941	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6942
6943	RTCPUID idCpu = RTMpCpuId();
6944	Log4Func(("HostCpuId=%u\n", idCpu));
6945
6946	/*
6947	* !!! IMPORTANT !!!
6948	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6949	*/
6950
6951	/* Save the guest state if necessary. */
6952	if ( fSaveGuestState
6953	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6954	{
6955	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6956	AssertRCReturn(rc, rc);
6957	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6958	}
6959
6960	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6961	if (CPUMIsGuestFPUStateActive(pVCpu))
6962	{
6963	/* We shouldn't reload CR0 without saving it first. */
6964	if (!fSaveGuestState)
6965	{
6966	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6967	AssertRCReturn(rc, rc);
6968	}
6969	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6970	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6971	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6972	}
6973
6974	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6975	#ifdef VBOX_STRICT
6976	if (CPUMIsHyperDebugStateActive(pVCpu))
6977	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6978	#endif
6979	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6980	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6981	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6982	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6983
6984	#if HC_ARCH_BITS == 64
6985	/* Restore host-state bits that VT-x only restores partially. */
6986	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6987	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6988	{
6989	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6990	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6991	}
6992	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6993	#endif
6994
6995	#if HC_ARCH_BITS == 64
6996	/* Restore the lazy host MSRs as we're leaving VT-x context. */
6997	if ( pVM->hm.s.fAllow64BitGuests
6998	&& pVCpu->hm.s.vmx.fLazyMsrs)
6999	{
7000	/* We shouldn't reload the guest MSRs without saving it first. */
7001	if (!fSaveGuestState)
7002	{
7003	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7004	AssertRCReturn(rc, rc);
7005	}
7006	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7007	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7008	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7009	}
7010	#endif
7011
7012	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7013	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7014
7015	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7016	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7017	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7018	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7019	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7020	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7021	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7022	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7023
7024	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7025
7026	/** @todo This partially defeats the purpose of having preemption hooks.
7027	* The problem is, deregistering the hooks should be moved to a place that
7028	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7029	* context.
7030	*/
7031	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7032	{
7033	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7034	AssertRCReturn(rc, rc);
7035
7036	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7037	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7038	}
7039	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7040	NOREF(idCpu);
7041
7042	return VINF_SUCCESS;
7043	}
7044
7045
7046	/**
7047	* Leaves the VT-x session.
7048	*
7049	* @returns VBox status code.
7050	* @param pVM The cross context VM structure.
7051	* @param pVCpu The cross context virtual CPU structure.
7052	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7053	* out-of-sync. Make sure to update the required fields
7054	* before using them.
7055	*
7056	* @remarks No-long-jmp zone!!!
7057	*/
7058	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7059	{
7060	HM_DISABLE_PREEMPT();
7061	HMVMX_ASSERT_CPU_SAFE();
7062	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7063	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7064
7065	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7066	and done this from the VMXR0ThreadCtxCallback(). */
7067	if (!pVCpu->hm.s.fLeaveDone)
7068	{
7069	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7070	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7071	pVCpu->hm.s.fLeaveDone = true;
7072	}
7073	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7074
7075	/*
7076	* !!! IMPORTANT !!!
7077	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7078	*/
7079
7080	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7081	/** @todo Deregistering here means we need to VMCLEAR always
7082	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7083	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7084	VMMR0ThreadCtxHookDisable(pVCpu);
7085
7086	/* Leave HM context. This takes care of local init (term). */
7087	int rc = HMR0LeaveCpu(pVCpu);
7088
7089	HM_RESTORE_PREEMPT();
7090	return rc;
7091	}
7092
7093
7094	/**
7095	* Does the necessary state syncing before doing a longjmp to ring-3.
7096	*
7097	* @returns VBox status code.
7098	* @param pVM The cross context VM structure.
7099	* @param pVCpu The cross context virtual CPU structure.
7100	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7101	* out-of-sync. Make sure to update the required fields
7102	* before using them.
7103	*
7104	* @remarks No-long-jmp zone!!!
7105	*/
7106	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7107	{
7108	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7109	}
7110
7111
7112	/**
7113	* Take necessary actions before going back to ring-3.
7114	*
7115	* An action requires us to go back to ring-3. This function does the necessary
7116	* steps before we can safely return to ring-3. This is not the same as longjmps
7117	* to ring-3, this is voluntary and prepares the guest so it may continue
7118	* executing outside HM (recompiler/IEM).
7119	*
7120	* @returns VBox status code.
7121	* @param pVM The cross context VM structure.
7122	* @param pVCpu The cross context virtual CPU structure.
7123	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7124	* out-of-sync. Make sure to update the required fields
7125	* before using them.
7126	* @param rcExit The reason for exiting to ring-3. Can be
7127	* VINF_VMM_UNKNOWN_RING3_CALL.
7128	*/
7129	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7130	{
7131	Assert(pVM);
7132	Assert(pVCpu);
7133	Assert(pMixedCtx);
7134	HMVMX_ASSERT_PREEMPT_SAFE();
7135
7136	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7137	{
7138	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7139	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7140	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7141	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7142	}
7143
7144	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7145	VMMRZCallRing3Disable(pVCpu);
7146	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7147
7148	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7149	if (pVCpu->hm.s.Event.fPending)
7150	{
7151	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7152	Assert(!pVCpu->hm.s.Event.fPending);
7153	}
7154
7155	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7156	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7157
7158	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7159	and if we're injecting an event we should have a TRPM trap pending. */
7160	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7161	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7162	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7163	#endif
7164
7165	/* Save guest state and restore host state bits. */
7166	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7167	AssertRCReturn(rc, rc);
7168	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7169	/* Thread-context hooks are unregistered at this point!!! */
7170
7171	/* Sync recompiler state. */
7172	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7173	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7174	\| CPUM_CHANGED_LDTR
7175	\| CPUM_CHANGED_GDTR
7176	\| CPUM_CHANGED_IDTR
7177	\| CPUM_CHANGED_TR
7178	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7179	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7180	if ( pVM->hm.s.fNestedPaging
7181	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7182	{
7183	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7184	}
7185
7186	Assert(!pVCpu->hm.s.fClearTrapFlag);
7187
7188	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7189	if (rcExit != VINF_EM_RAW_INTERRUPT)
7190	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7191
7192	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7193
7194	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7195	VMMRZCallRing3RemoveNotification(pVCpu);
7196	VMMRZCallRing3Enable(pVCpu);
7197
7198	return rc;
7199	}
7200
7201
7202	/**
7203	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7204	* longjump to ring-3 and possibly get preempted.
7205	*
7206	* @returns VBox status code.
7207	* @param pVCpu The cross context virtual CPU structure.
7208	* @param enmOperation The operation causing the ring-3 longjump.
7209	* @param pvUser Opaque pointer to the guest-CPU context. The data
7210	* may be out-of-sync. Make sure to update the required
7211	* fields before using them.
7212	*/
7213	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7214	{
7215	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7216	{
7217	/*
7218	* !!! IMPORTANT !!!
7219	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7220	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7221	*/
7222	VMMRZCallRing3RemoveNotification(pVCpu);
7223	VMMRZCallRing3Disable(pVCpu);
7224	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7225	RTThreadPreemptDisable(&PreemptState);
7226
7227	PVM pVM = pVCpu->CTX_SUFF(pVM);
7228	if (CPUMIsGuestFPUStateActive(pVCpu))
7229	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7230
7231	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7232
7233	#if HC_ARCH_BITS == 64
7234	/* Restore host-state bits that VT-x only restores partially. */
7235	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7236	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7237	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7238	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7239
7240	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7241	if ( pVM->hm.s.fAllow64BitGuests
7242	&& pVCpu->hm.s.vmx.fLazyMsrs)
7243	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7244	#endif
7245	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7246	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7247	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7248	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7249	{
7250	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7251	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7252	}
7253
7254	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7255	VMMR0ThreadCtxHookDisable(pVCpu);
7256	HMR0LeaveCpu(pVCpu);
7257	RTThreadPreemptRestore(&PreemptState);
7258	return VINF_SUCCESS;
7259	}
7260
7261	Assert(pVCpu);
7262	Assert(pvUser);
7263	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7264	HMVMX_ASSERT_PREEMPT_SAFE();
7265
7266	VMMRZCallRing3Disable(pVCpu);
7267	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7268
7269	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7270	enmOperation));
7271
7272	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7273	AssertRCReturn(rc, rc);
7274
7275	VMMRZCallRing3Enable(pVCpu);
7276	return VINF_SUCCESS;
7277	}
7278
7279
7280	/**
7281	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7282	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7283	*
7284	* @param pVCpu The cross context virtual CPU structure.
7285	*/
7286	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7287	{
7288	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7289	{
7290	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7291	{
7292	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7293	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7294	AssertRC(rc);
7295	Log4(("Setup interrupt-window exiting\n"));
7296	}
7297	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7298	}
7299
7300
7301	/**
7302	* Clears the interrupt-window exiting control in the VMCS.
7303	*
7304	* @param pVCpu The cross context virtual CPU structure.
7305	*/
7306	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7307	{
7308	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7309	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7310	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7311	AssertRC(rc);
7312	Log4(("Cleared interrupt-window exiting\n"));
7313	}
7314
7315
7316	/**
7317	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7318	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7319	*
7320	* @param pVCpu The cross context virtual CPU structure.
7321	*/
7322	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7323	{
7324	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7325	{
7326	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7327	{
7328	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7329	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7330	AssertRC(rc);
7331	Log4(("Setup NMI-window exiting\n"));
7332	}
7333	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7334	}
7335
7336
7337	/**
7338	* Clears the NMI-window exiting control in the VMCS.
7339	*
7340	* @param pVCpu The cross context virtual CPU structure.
7341	*/
7342	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7343	{
7344	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7345	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7346	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7347	AssertRC(rc);
7348	Log4(("Cleared NMI-window exiting\n"));
7349	}
7350
7351
7352	/**
7353	* Evaluates the event to be delivered to the guest and sets it as the pending
7354	* event.
7355	*
7356	* @param pVCpu The cross context virtual CPU structure.
7357	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7358	* out-of-sync. Make sure to update the required fields
7359	* before using them.
7360	*/
7361	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7362	{
7363	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7364	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7365	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7366	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7367	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7368
7369	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7370	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7371	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7372	Assert(!TRPMHasTrap(pVCpu));
7373
7374	/*
7375	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7376	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7377	*/
7378	/** @todo SMI. SMIs take priority over NMIs. */
7379	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7380	{
7381	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7382	if ( !pVCpu->hm.s.Event.fPending
7383	&& !fBlockNmi
7384	&& !fBlockSti
7385	&& !fBlockMovSS)
7386	{
7387	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7388	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7389	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7390
7391	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7392	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7393	}
7394	else
7395	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7396	}
7397	/*
7398	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7399	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7400	*/
7401	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7402	&& !pVCpu->hm.s.fSingleInstruction)
7403	{
7404	Assert(!DBGFIsStepping(pVCpu));
7405	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7406	AssertRC(rc);
7407	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7408	if ( !pVCpu->hm.s.Event.fPending
7409	&& !fBlockInt
7410	&& !fBlockSti
7411	&& !fBlockMovSS)
7412	{
7413	uint8_t u8Interrupt;
7414	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7415	if (RT_SUCCESS(rc))
7416	{
7417	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7418	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7419	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7420
7421	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7422	}
7423	else
7424	{
7425	/** @todo Does this actually happen? If not turn it into an assertion. */
7426	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7427	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7428	}
7429	}
7430	else
7431	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7432	}
7433	}
7434
7435
7436	/**
7437	* Sets a pending-debug exception to be delivered to the guest if the guest is
7438	* single-stepping in the VMCS.
7439	*
7440	* @param pVCpu The cross context virtual CPU structure.
7441	*/
7442	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7443	{
7444	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7445	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7446	AssertRC(rc);
7447	}
7448
7449
7450	/**
7451	* Injects any pending events into the guest if the guest is in a state to
7452	* receive them.
7453	*
7454	* @returns Strict VBox status code (informational status codes included).
7455	* @param pVCpu The cross context virtual CPU structure.
7456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7457	* out-of-sync. Make sure to update the required fields
7458	* before using them.
7459	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7460	* return VINF_EM_DBG_STEPPED if the event was
7461	* dispatched directly.
7462	*/
7463	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7464	{
7465	HMVMX_ASSERT_PREEMPT_SAFE();
7466	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7467
7468	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7469	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7470	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7471	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7472
7473	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7474	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7475	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7476	Assert(!TRPMHasTrap(pVCpu));
7477
7478	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7479	if (pVCpu->hm.s.Event.fPending)
7480	{
7481	/*
7482	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7483	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7484	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7485	*
7486	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7487	*/
7488	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7489	#ifdef VBOX_STRICT
7490	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7491	{
7492	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7493	Assert(!fBlockInt);
7494	Assert(!fBlockSti);
7495	Assert(!fBlockMovSS);
7496	}
7497	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7498	{
7499	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7500	Assert(!fBlockSti);
7501	Assert(!fBlockMovSS);
7502	Assert(!fBlockNmi);
7503	}
7504	#endif
7505	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7506	(uint8_t)uIntType));
7507	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7508	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7509	fStepping, &uIntrState);
7510	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7511
7512	/* Update the interruptibility-state as it could have been changed by
7513	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7514	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7515	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7516
7517	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7518	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7519	else
7520	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7521	}
7522
7523	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7524	if ( fBlockSti
7525	\|\| fBlockMovSS)
7526	{
7527	if (!pVCpu->hm.s.fSingleInstruction)
7528	{
7529	/*
7530	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7531	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7532	* See Intel spec. 27.3.4 "Saving Non-Register State".
7533	*/
7534	Assert(!DBGFIsStepping(pVCpu));
7535	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7536	AssertRCReturn(rc2, rc2);
7537	if (pMixedCtx->eflags.Bits.u1TF)
7538	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7539	}
7540	else if (pMixedCtx->eflags.Bits.u1TF)
7541	{
7542	/*
7543	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7544	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7545	*/
7546	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7547	uIntrState = 0;
7548	}
7549	}
7550
7551	/*
7552	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7553	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7554	*/
7555	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7556	AssertRC(rc2);
7557
7558	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7559	NOREF(fBlockMovSS); NOREF(fBlockSti);
7560	return rcStrict;
7561	}
7562
7563
7564	/**
7565	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7566	*
7567	* @param pVCpu The cross context virtual CPU structure.
7568	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7569	* out-of-sync. Make sure to update the required fields
7570	* before using them.
7571	*/
7572	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7573	{
7574	NOREF(pMixedCtx);
7575	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7576	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7577	}
7578
7579
7580	/**
7581	* Injects a double-fault (\#DF) exception into the VM.
7582	*
7583	* @returns Strict VBox status code (informational status code included).
7584	* @param pVCpu The cross context virtual CPU structure.
7585	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7586	* out-of-sync. Make sure to update the required fields
7587	* before using them.
7588	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7589	* and should return VINF_EM_DBG_STEPPED if the event
7590	* is injected directly (register modified by us, not
7591	* by hardware on VM-entry).
7592	* @param puIntrState Pointer to the current guest interruptibility-state.
7593	* This interruptibility-state will be updated if
7594	* necessary. This cannot not be NULL.
7595	*/
7596	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7597	{
7598	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7599	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7600	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7601	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7602	fStepping, puIntrState);
7603	}
7604
7605
7606	/**
7607	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7608	*
7609	* @param pVCpu The cross context virtual CPU structure.
7610	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7611	* out-of-sync. Make sure to update the required fields
7612	* before using them.
7613	*/
7614	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7615	{
7616	NOREF(pMixedCtx);
7617	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7618	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7619	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7620	}
7621
7622
7623	/**
7624	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7625	*
7626	* @param pVCpu The cross context virtual CPU structure.
7627	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7628	* out-of-sync. Make sure to update the required fields
7629	* before using them.
7630	* @param cbInstr The value of RIP that is to be pushed on the guest
7631	* stack.
7632	*/
7633	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7634	{
7635	NOREF(pMixedCtx);
7636	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7637	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7638	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7639	}
7640
7641
7642	/**
7643	* Injects a general-protection (\#GP) fault into the VM.
7644	*
7645	* @returns Strict VBox status code (informational status code included).
7646	* @param pVCpu The cross context virtual CPU structure.
7647	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7648	* out-of-sync. Make sure to update the required fields
7649	* before using them.
7650	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7651	* mode, i.e. in real-mode it's not valid).
7652	* @param u32ErrorCode The error code associated with the \#GP.
7653	* @param fStepping Whether we're running in
7654	* hmR0VmxRunGuestCodeStep() and should return
7655	* VINF_EM_DBG_STEPPED if the event is injected
7656	* directly (register modified by us, not by
7657	* hardware on VM-entry).
7658	* @param puIntrState Pointer to the current guest interruptibility-state.
7659	* This interruptibility-state will be updated if
7660	* necessary. This cannot not be NULL.
7661	*/
7662	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7663	bool fStepping, uint32_t *puIntrState)
7664	{
7665	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7666	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7667	if (fErrorCodeValid)
7668	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7669	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7670	fStepping, puIntrState);
7671	}
7672
7673
7674	/**
7675	* Sets a general-protection (\#GP) exception as pending-for-injection into the
7676	* VM.
7677	*
7678	* @param pVCpu The cross context virtual CPU structure.
7679	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7680	* out-of-sync. Make sure to update the required fields
7681	* before using them.
7682	* @param u32ErrorCode The error code associated with the \#GP.
7683	*/
7684	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7685	{
7686	NOREF(pMixedCtx);
7687	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7688	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7689	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7690	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
7691	}
7692
7693
7694	/**
7695	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7696	*
7697	* @param pVCpu The cross context virtual CPU structure.
7698	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7699	* out-of-sync. Make sure to update the required fields
7700	* before using them.
7701	* @param uVector The software interrupt vector number.
7702	* @param cbInstr The value of RIP that is to be pushed on the guest
7703	* stack.
7704	*/
7705	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7706	{
7707	NOREF(pMixedCtx);
7708	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7709	if ( uVector == X86_XCPT_BP
7710	\|\| uVector == X86_XCPT_OF)
7711	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7712	else
7713	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7714	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7715	}
7716
7717
7718	/**
7719	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7720	* stack.
7721	*
7722	* @returns Strict VBox status code (information status code included).
7723	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7724	* @param pVM The cross context VM structure.
7725	* @param pMixedCtx Pointer to the guest-CPU context.
7726	* @param uValue The value to push to the guest stack.
7727	*/
7728	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7729	{
7730	/*
7731	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7732	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7733	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7734	*/
7735	if (pMixedCtx->sp == 1)
7736	return VINF_EM_RESET;
7737	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7738	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7739	AssertRC(rc);
7740	return rc;
7741	}
7742
7743
7744	/**
7745	* Injects an event into the guest upon VM-entry by updating the relevant fields
7746	* in the VM-entry area in the VMCS.
7747	*
7748	* @returns Strict VBox status code (informational error codes included).
7749	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7750	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7751	*
7752	* @param pVCpu The cross context virtual CPU structure.
7753	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7754	* be out-of-sync. Make sure to update the required
7755	* fields before using them.
7756	* @param u64IntInfo The VM-entry interruption-information field.
7757	* @param cbInstr The VM-entry instruction length in bytes (for
7758	* software interrupts, exceptions and privileged
7759	* software exceptions).
7760	* @param u32ErrCode The VM-entry exception error code.
7761	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7762	* @param puIntrState Pointer to the current guest interruptibility-state.
7763	* This interruptibility-state will be updated if
7764	* necessary. This cannot not be NULL.
7765	* @param fStepping Whether we're running in
7766	* hmR0VmxRunGuestCodeStep() and should return
7767	* VINF_EM_DBG_STEPPED if the event is injected
7768	* directly (register modified by us, not by
7769	* hardware on VM-entry).
7770	*
7771	* @remarks Requires CR0!
7772	* @remarks No-long-jump zone!!!
7773	*/
7774	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7775	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7776	uint32_t *puIntrState)
7777	{
7778	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7779	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7780	Assert(puIntrState);
7781	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7782
7783	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7784	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7785
7786	#ifdef VBOX_STRICT
7787	/* Validate the error-code-valid bit for hardware exceptions. */
7788	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7789	{
7790	switch (uVector)
7791	{
7792	case X86_XCPT_PF:
7793	case X86_XCPT_DF:
7794	case X86_XCPT_TS:
7795	case X86_XCPT_NP:
7796	case X86_XCPT_SS:
7797	case X86_XCPT_GP:
7798	case X86_XCPT_AC:
7799	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7800	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7801	/* fallthru */
7802	default:
7803	break;
7804	}
7805	}
7806	#endif
7807
7808	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7809	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7810	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7811
7812	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7813
7814	/* We require CR0 to check if the guest is in real-mode. */
7815	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7816	AssertRCReturn(rc, rc);
7817
7818	/*
7819	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7820	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7821	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7822	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7823	*/
7824	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7825	{
7826	PVM pVM = pVCpu->CTX_SUFF(pVM);
7827	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7828	{
7829	Assert(PDMVmmDevHeapIsEnabled(pVM));
7830	Assert(pVM->hm.s.vmx.pRealModeTSS);
7831
7832	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7833	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7834	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7835	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7836	AssertRCReturn(rc, rc);
7837	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7838
7839	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7840	size_t const cbIdtEntry = sizeof(X86IDTR16);
7841	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7842	{
7843	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7844	if (uVector == X86_XCPT_DF)
7845	return VINF_EM_RESET;
7846
7847	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7848	if (uVector == X86_XCPT_GP)
7849	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7850
7851	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7852	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7853	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
7854	fStepping, puIntrState);
7855	}
7856
7857	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7858	uint16_t uGuestIp = pMixedCtx->ip;
7859	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7860	{
7861	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7862	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7863	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7864	}
7865	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7866	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7867
7868	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7869	X86IDTR16 IdtEntry;
7870	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7871	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7872	AssertRCReturn(rc, rc);
7873
7874	/* Construct the stack frame for the interrupt/exception handler. */
7875	VBOXSTRICTRC rcStrict;
7876	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7877	if (rcStrict == VINF_SUCCESS)
7878	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7879	if (rcStrict == VINF_SUCCESS)
7880	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7881
7882	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7883	if (rcStrict == VINF_SUCCESS)
7884	{
7885	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7886	pMixedCtx->rip = IdtEntry.offSel;
7887	pMixedCtx->cs.Sel = IdtEntry.uSel;
7888	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7889	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7890	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7891	&& uVector == X86_XCPT_PF)
7892	pMixedCtx->cr2 = GCPtrFaultAddress;
7893
7894	/* If any other guest-state bits are changed here, make sure to update
7895	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7896	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7897	\| HM_CHANGED_GUEST_RIP
7898	\| HM_CHANGED_GUEST_RFLAGS
7899	\| HM_CHANGED_GUEST_RSP);
7900
7901	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7902	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7903	{
7904	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7905	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7906	Log4(("Clearing inhibition due to STI.\n"));
7907	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7908	}
7909	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7910	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7911
7912	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7913	it, if we are returning to ring-3 before executing guest code. */
7914	pVCpu->hm.s.Event.fPending = false;
7915
7916	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7917	if (fStepping)
7918	rcStrict = VINF_EM_DBG_STEPPED;
7919	}
7920	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7921	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7922	return rcStrict;
7923	}
7924
7925	/*
7926	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7927	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7928	*/
7929	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7930	}
7931
7932	/* Validate. */
7933	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7934	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7935	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7936
7937	/* Inject. */
7938	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7939	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7940	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7941	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7942
7943	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7944	&& uVector == X86_XCPT_PF)
7945	pMixedCtx->cr2 = GCPtrFaultAddress;
7946
7947	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7948	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7949
7950	AssertRCReturn(rc, rc);
7951	return VINF_SUCCESS;
7952	}
7953
7954
7955	/**
7956	* Clears the interrupt-window exiting control in the VMCS and if necessary
7957	* clears the current event in the VMCS as well.
7958	*
7959	* @returns VBox status code.
7960	* @param pVCpu The cross context virtual CPU structure.
7961	*
7962	* @remarks Use this function only to clear events that have not yet been
7963	* delivered to the guest but are injected in the VMCS!
7964	* @remarks No-long-jump zone!!!
7965	*/
7966	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7967	{
7968	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7969
7970	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7971	hmR0VmxClearIntWindowExitVmcs(pVCpu);
7972
7973	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7974	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7975	}
7976
7977
7978	/**
7979	* Enters the VT-x session.
7980	*
7981	* @returns VBox status code.
7982	* @param pVM The cross context VM structure.
7983	* @param pVCpu The cross context virtual CPU structure.
7984	* @param pCpu Pointer to the CPU info struct.
7985	*/
7986	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7987	{
7988	AssertPtr(pVM);
7989	AssertPtr(pVCpu);
7990	Assert(pVM->hm.s.vmx.fSupported);
7991	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7992	NOREF(pCpu); NOREF(pVM);
7993
7994	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7995	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7996
7997	#ifdef VBOX_STRICT
7998	/* Make sure we're in VMX root mode. */
7999	RTCCUINTREG u32HostCR4 = ASMGetCR4();
8000	if (!(u32HostCR4 & X86_CR4_VMXE))
8001	{
8002	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8003	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8004	}
8005	#endif
8006
8007	/*
8008	* Load the VCPU's VMCS as the current (and active) one.
8009	*/
8010	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8011	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8012	if (RT_FAILURE(rc))
8013	return rc;
8014
8015	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8016	pVCpu->hm.s.fLeaveDone = false;
8017	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8018
8019	return VINF_SUCCESS;
8020	}
8021
8022
8023	/**
8024	* The thread-context callback (only on platforms which support it).
8025	*
8026	* @param enmEvent The thread-context event.
8027	* @param pVCpu The cross context virtual CPU structure.
8028	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8029	* @thread EMT(pVCpu)
8030	*/
8031	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8032	{
8033	NOREF(fGlobalInit);
8034
8035	switch (enmEvent)
8036	{
8037	case RTTHREADCTXEVENT_OUT:
8038	{
8039	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8040	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8041	VMCPU_ASSERT_EMT(pVCpu);
8042
8043	PVM pVM = pVCpu->CTX_SUFF(pVM);
8044	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8045
8046	/* No longjmps (logger flushes, locks) in this fragile context. */
8047	VMMRZCallRing3Disable(pVCpu);
8048	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8049
8050	/*
8051	* Restore host-state (FPU, debug etc.)
8052	*/
8053	if (!pVCpu->hm.s.fLeaveDone)
8054	{
8055	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8056	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8057	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8058	pVCpu->hm.s.fLeaveDone = true;
8059	}
8060
8061	/* Leave HM context, takes care of local init (term). */
8062	int rc = HMR0LeaveCpu(pVCpu);
8063	AssertRC(rc); NOREF(rc);
8064
8065	/* Restore longjmp state. */
8066	VMMRZCallRing3Enable(pVCpu);
8067	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8068	break;
8069	}
8070
8071	case RTTHREADCTXEVENT_IN:
8072	{
8073	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8074	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8075	VMCPU_ASSERT_EMT(pVCpu);
8076
8077	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8078	VMMRZCallRing3Disable(pVCpu);
8079	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8080
8081	/* Initialize the bare minimum state required for HM. This takes care of
8082	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8083	int rc = HMR0EnterCpu(pVCpu);
8084	AssertRC(rc);
8085	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8086
8087	/* Load the active VMCS as the current one. */
8088	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8089	{
8090	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8091	AssertRC(rc); NOREF(rc);
8092	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8093	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8094	}
8095	pVCpu->hm.s.fLeaveDone = false;
8096
8097	/* Restore longjmp state. */
8098	VMMRZCallRing3Enable(pVCpu);
8099	break;
8100	}
8101
8102	default:
8103	break;
8104	}
8105	}
8106
8107
8108	/**
8109	* Saves the host state in the VMCS host-state.
8110	* Sets up the VM-exit MSR-load area.
8111	*
8112	* The CPU state will be loaded from these fields on every successful VM-exit.
8113	*
8114	* @returns VBox status code.
8115	* @param pVM The cross context VM structure.
8116	* @param pVCpu The cross context virtual CPU structure.
8117	*
8118	* @remarks No-long-jump zone!!!
8119	*/
8120	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8121	{
8122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8123
8124	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8125	return VINF_SUCCESS;
8126
8127	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8128	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8129
8130	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8131	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8132
8133	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8134	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8135
8136	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8137	return rc;
8138	}
8139
8140
8141	/**
8142	* Saves the host state in the VMCS host-state.
8143	*
8144	* @returns VBox status code.
8145	* @param pVM The cross context VM structure.
8146	* @param pVCpu The cross context virtual CPU structure.
8147	*
8148	* @remarks No-long-jump zone!!!
8149	*/
8150	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8151	{
8152	AssertPtr(pVM);
8153	AssertPtr(pVCpu);
8154
8155	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8156
8157	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8158	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8159	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8160	return hmR0VmxSaveHostState(pVM, pVCpu);
8161	}
8162
8163
8164	/**
8165	* Loads the guest state into the VMCS guest-state area.
8166	*
8167	* The will typically be done before VM-entry when the guest-CPU state and the
8168	* VMCS state may potentially be out of sync.
8169	*
8170	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8171	* VM-entry controls.
8172	* Sets up the appropriate VMX non-root function to execute guest code based on
8173	* the guest CPU mode.
8174	*
8175	* @returns VBox status code.
8176	* @param pVM The cross context VM structure.
8177	* @param pVCpu The cross context virtual CPU structure.
8178	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8179	* out-of-sync. Make sure to update the required fields
8180	* before using them.
8181	*
8182	* @remarks No-long-jump zone!!!
8183	*/
8184	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8185	{
8186	AssertPtr(pVM);
8187	AssertPtr(pVCpu);
8188	AssertPtr(pMixedCtx);
8189	HMVMX_ASSERT_PREEMPT_SAFE();
8190
8191	VMMRZCallRing3Disable(pVCpu);
8192	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8193
8194	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8195
8196	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8197
8198	/* Determine real-on-v86 mode. */
8199	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8200	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8201	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8202	{
8203	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8204	}
8205
8206	/*
8207	* Load the guest-state into the VMCS.
8208	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8209	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8210	*/
8211	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8212	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8213
8214	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8215	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8216	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8217
8218	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8219	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8220	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8221
8222	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8223	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8224
8225	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8226	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8227
8228	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8229	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8230	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8231
8232	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8233	determine we don't have to swap EFER after all. */
8234	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8235	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8236
8237	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8238	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8239
8240	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8241	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8242
8243	/*
8244	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8245	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8246	*/
8247	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8248	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8249
8250	/* Clear any unused and reserved bits. */
8251	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8252
8253	VMMRZCallRing3Enable(pVCpu);
8254
8255	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8256	return rc;
8257	}
8258
8259
8260	/**
8261	* Loads the state shared between the host and guest into the VMCS.
8262	*
8263	* @param pVM The cross context VM structure.
8264	* @param pVCpu The cross context virtual CPU structure.
8265	* @param pCtx Pointer to the guest-CPU context.
8266	*
8267	* @remarks No-long-jump zone!!!
8268	*/
8269	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8270	{
8271	NOREF(pVM);
8272
8273	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8274	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8275
8276	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8277	{
8278	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8279	AssertRC(rc);
8280	}
8281
8282	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8283	{
8284	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8285	AssertRC(rc);
8286
8287	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8288	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8289	{
8290	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8291	AssertRC(rc);
8292	}
8293	}
8294
8295	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8296	{
8297	#if HC_ARCH_BITS == 64
8298	if (pVM->hm.s.fAllow64BitGuests)
8299	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8300	#endif
8301	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8302	}
8303
8304	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8305	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8306	{
8307	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8308	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8309	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8310	AssertRC(rc);
8311	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8312	}
8313
8314	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8315	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8316	}
8317
8318
8319	/**
8320	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8321	*
8322	* @param pVM The cross context VM structure.
8323	* @param pVCpu The cross context virtual CPU structure.
8324	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8325	* out-of-sync. Make sure to update the required fields
8326	* before using them.
8327	*/
8328	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8329	{
8330	HMVMX_ASSERT_PREEMPT_SAFE();
8331
8332	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8333	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8334	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8335	#endif
8336
8337	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8338	{
8339	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8340	AssertRC(rc);
8341	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8342	}
8343	else if (HMCPU_CF_VALUE(pVCpu))
8344	{
8345	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8346	AssertRC(rc);
8347	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8348	}
8349
8350	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8351	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8352	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8353	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8354	}
8355
8356
8357	/**
8358	* Does the preparations before executing guest code in VT-x.
8359	*
8360	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8361	* recompiler/IEM. We must be cautious what we do here regarding committing
8362	* guest-state information into the VMCS assuming we assuredly execute the
8363	* guest in VT-x mode.
8364	*
8365	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8366	* the common-state (TRPM/forceflags), we must undo those changes so that the
8367	* recompiler/IEM can (and should) use them when it resumes guest execution.
8368	* Otherwise such operations must be done when we can no longer exit to ring-3.
8369	*
8370	* @returns Strict VBox status code.
8371	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8372	* have been disabled.
8373	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8374	* double-fault into the guest.
8375	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8376	* dispatched directly.
8377	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8378	*
8379	* @param pVM The cross context VM structure.
8380	* @param pVCpu The cross context virtual CPU structure.
8381	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8382	* out-of-sync. Make sure to update the required fields
8383	* before using them.
8384	* @param pVmxTransient Pointer to the VMX transient structure.
8385	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8386	* us ignore some of the reasons for returning to
8387	* ring-3, and return VINF_EM_DBG_STEPPED if event
8388	* dispatching took place.
8389	*/
8390	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8391	{
8392	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8393
8394	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8395	PGMRZDynMapFlushAutoSet(pVCpu);
8396	#endif
8397
8398	/* Check force flag actions that might require us to go back to ring-3. */
8399	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8400	if (rcStrict == VINF_SUCCESS)
8401	{ /* FFs doesn't get set all the time. */ }
8402	else
8403	return rcStrict;
8404
8405	#ifndef IEM_VERIFICATION_MODE_FULL
8406	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8407	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8408	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8409	{
8410	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8411	RTGCPHYS GCPhysApicBase;
8412	GCPhysApicBase = pMixedCtx->msrApicBase;
8413	GCPhysApicBase &= PAGE_BASE_GC_MASK;
8414
8415	/* Unalias any existing mapping. */
8416	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8417	AssertRCReturn(rc, rc);
8418
8419	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8420	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8421	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8422	AssertRCReturn(rc, rc);
8423
8424	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8425	}
8426	#endif /* !IEM_VERIFICATION_MODE_FULL */
8427
8428	if (TRPMHasTrap(pVCpu))
8429	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8430	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8431
8432	/*
8433	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8434	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8435	*/
8436	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8437	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8438	{ /* likely */ }
8439	else
8440	{
8441	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8442	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8443	return rcStrict;
8444	}
8445
8446	/*
8447	* Load the guest state bits, we can handle longjmps/getting preempted here.
8448	*
8449	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8450	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8451	* Hence, this needs to be done -after- injection of events.
8452	*/
8453	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8454
8455	/*
8456	* No longjmps to ring-3 from this point on!!!
8457	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8458	* This also disables flushing of the R0-logger instance (if any).
8459	*/
8460	VMMRZCallRing3Disable(pVCpu);
8461
8462	/*
8463	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8464	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8465	*
8466	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8467	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8468	*
8469	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8470	* executing guest code.
8471	*/
8472	pVmxTransient->fEFlags = ASMIntDisableFlags();
8473
8474	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8475	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8476	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8477	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8478	{
8479	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8480	{
8481	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8482	pVCpu->hm.s.Event.fPending = false;
8483
8484	return VINF_SUCCESS;
8485	}
8486
8487	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8488	rcStrict = VINF_EM_RAW_INTERRUPT;
8489	}
8490	else
8491	{
8492	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8493	rcStrict = VINF_EM_RAW_TO_R3;
8494	}
8495
8496	ASMSetFlags(pVmxTransient->fEFlags);
8497	VMMRZCallRing3Enable(pVCpu);
8498
8499	return rcStrict;
8500	}
8501
8502
8503	/**
8504	* Prepares to run guest code in VT-x and we've committed to doing so. This
8505	* means there is no backing out to ring-3 or anywhere else at this
8506	* point.
8507	*
8508	* @param pVM The cross context VM structure.
8509	* @param pVCpu The cross context virtual CPU structure.
8510	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8511	* out-of-sync. Make sure to update the required fields
8512	* before using them.
8513	* @param pVmxTransient Pointer to the VMX transient structure.
8514	*
8515	* @remarks Called with preemption disabled.
8516	* @remarks No-long-jump zone!!!
8517	*/
8518	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8519	{
8520	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8521	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8522	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8523
8524	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8525	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8526
8527	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8528	if (!CPUMIsGuestFPUStateActive(pVCpu))
8529	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8530	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8531	#endif
8532
8533	if ( pVCpu->hm.s.fPreloadGuestFpu
8534	&& !CPUMIsGuestFPUStateActive(pVCpu))
8535	{
8536	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8537	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8538	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8539	}
8540
8541	/*
8542	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8543	*/
8544	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8545	&& pVCpu->hm.s.vmx.cMsrs > 0)
8546	{
8547	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8548	}
8549
8550	/*
8551	* Load the host state bits as we may've been preempted (only happens when
8552	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8553	*/
8554	/** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8555	* any effect to the host state needing to be saved? */
8556	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8557	{
8558	/* This ASSUMES that pfnStartVM has been set up already. */
8559	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8560	AssertRC(rc);
8561	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8562	}
8563	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8564
8565	/*
8566	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8567	*/
8568	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8569	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8570	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8571
8572	/* Store status of the shared guest-host state at the time of VM-entry. */
8573	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8574	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8575	{
8576	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8577	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8578	}
8579	else
8580	#endif
8581	{
8582	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8583	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8584	}
8585	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8586
8587	/*
8588	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8589	*/
8590	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8591	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8592
8593	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8594	RTCPUID idCurrentCpu = pCpu->idCpu;
8595	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8596	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8597	{
8598	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8599	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8600	}
8601
8602	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8603	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8604	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8605	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8606
8607	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8608
8609	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8610	to start executing. */
8611
8612	/*
8613	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8614	*/
8615	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8616	{
8617	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8618	{
8619	bool fMsrUpdated;
8620	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8621	AssertRC(rc2);
8622	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8623
8624	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8625	&fMsrUpdated);
8626	AssertRC(rc2);
8627	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8628
8629	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8630	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8631	}
8632	else
8633	{
8634	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8635	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8636	}
8637	}
8638
8639	#ifdef VBOX_STRICT
8640	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8641	hmR0VmxCheckHostEferMsr(pVCpu);
8642	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8643	#endif
8644	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8645	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8646	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8647	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8648	#endif
8649	}
8650
8651
8652	/**
8653	* Performs some essential restoration of state after running guest code in
8654	* VT-x.
8655	*
8656	* @param pVM The cross context VM structure.
8657	* @param pVCpu The cross context virtual CPU structure.
8658	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8659	* out-of-sync. Make sure to update the required fields
8660	* before using them.
8661	* @param pVmxTransient Pointer to the VMX transient structure.
8662	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8663	*
8664	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8665	*
8666	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8667	* unconditionally when it is safe to do so.
8668	*/
8669	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8670	{
8671	NOREF(pVM);
8672
8673	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8674
8675	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8676	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8677	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8678	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8679	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8680	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8681
8682	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8683	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8684
8685	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8686	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8687	Assert(!ASMIntAreEnabled());
8688	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8689
8690	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8691	if (CPUMIsGuestFPUStateActive(pVCpu))
8692	{
8693	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8694	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8695	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8696	}
8697	#endif
8698
8699	#if HC_ARCH_BITS == 64
8700	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8701	#endif
8702	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8703	#ifdef VBOX_STRICT
8704	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8705	#endif
8706	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8707	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8708
8709	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8710	uint32_t uExitReason;
8711	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8712	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8713	AssertRC(rc);
8714	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8715	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8716
8717	/* Update the VM-exit history array. */
8718	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8719
8720	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8721	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8722	{
8723	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8724	pVmxTransient->fVMEntryFailed));
8725	return;
8726	}
8727
8728	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8729	{
8730	/** @todo We can optimize this by only syncing with our force-flags when
8731	* really needed and keeping the VMCS state as it is for most
8732	* VM-exits. */
8733	/* Update the guest interruptibility-state from the VMCS. */
8734	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8735
8736	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8737	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8738	AssertRC(rc);
8739	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8740	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8741	AssertRC(rc);
8742	#endif
8743
8744	/*
8745	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8746	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8747	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8748	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8749	*/
8750	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8751	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8752	{
8753	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8754	AssertRC(rc);
8755	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8756	}
8757	}
8758	}
8759
8760
8761	/**
8762	* Runs the guest code using VT-x the normal way.
8763	*
8764	* @returns VBox status code.
8765	* @param pVM The cross context VM structure.
8766	* @param pVCpu The cross context virtual CPU structure.
8767	* @param pCtx Pointer to the guest-CPU context.
8768	*
8769	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8770	*/
8771	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8772	{
8773	VMXTRANSIENT VmxTransient;
8774	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8775	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8776	uint32_t cLoops = 0;
8777
8778	for (;; cLoops++)
8779	{
8780	Assert(!HMR0SuspendPending());
8781	HMVMX_ASSERT_CPU_SAFE();
8782
8783	/* Preparatory work for running guest code, this may force us to return
8784	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8785	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8786	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8787	if (rcStrict != VINF_SUCCESS)
8788	break;
8789
8790	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8791	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8792	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8793
8794	/* Restore any residual host-state and save any bits shared between host
8795	and guest into the guest-CPU state. Re-enables interrupts! */
8796	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8797
8798	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8799	if (RT_SUCCESS(rcRun))
8800	{ /* very likely */ }
8801	else
8802	{
8803	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8804	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8805	return rcRun;
8806	}
8807
8808	/* Profile the VM-exit. */
8809	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8811	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8812	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8813	HMVMX_START_EXIT_DISPATCH_PROF();
8814
8815	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8816	if (RT_LIKELY(!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8817	{ /* likely */ }
8818	else
8819	{
8820	hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8821	hmR0VmxSaveGuestState(pVCpu, pCtx);
8822	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8823	}
8824
8825	/* Handle the VM-exit. */
8826	#ifdef HMVMX_USE_FUNCTION_TABLE
8827	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8828	#else
8829	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8830	#endif
8831	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8832	if (rcStrict == VINF_SUCCESS)
8833	{
8834	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8835	continue; /* likely */
8836	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8837	rcStrict = VINF_EM_RAW_INTERRUPT;
8838	}
8839	break;
8840	}
8841
8842	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8843	return rcStrict;
8844	}
8845
8846
8847	/**
8848	* Single steps guest code using VT-x.
8849	*
8850	* @returns Strict VBox status code.
8851	* @param pVM The cross context VM structure.
8852	* @param pVCpu The cross context virtual CPU structure.
8853	* @param pCtx Pointer to the guest-CPU context.
8854	*
8855	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8856	*/
8857	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8858	{
8859	VMXTRANSIENT VmxTransient;
8860	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8861	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8862	uint32_t cLoops = 0;
8863	uint16_t uCsStart = pCtx->cs.Sel;
8864	uint64_t uRipStart = pCtx->rip;
8865
8866	for (;; cLoops++)
8867	{
8868	Assert(!HMR0SuspendPending());
8869	HMVMX_ASSERT_CPU_SAFE();
8870
8871	/* Preparatory work for running guest code, this may force us to return
8872	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8873	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8874	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8875	if (rcStrict != VINF_SUCCESS)
8876	break;
8877
8878	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8879	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8880	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8881
8882	/* Restore any residual host-state and save any bits shared between host
8883	and guest into the guest-CPU state. Re-enables interrupts! */
8884	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8885
8886	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8887	if (RT_SUCCESS(rcRun))
8888	{ /* very likely */ }
8889	else
8890	{
8891	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8892	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8893	return rcRun;
8894	}
8895
8896	/* Profile the VM-exit. */
8897	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8899	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8900	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8901	HMVMX_START_EXIT_DISPATCH_PROF();
8902
8903	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8904	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
8905	{ /* more likely */ }
8906	else
8907	{
8908	hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8909	hmR0VmxSaveGuestState(pVCpu, pCtx);
8910	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8911	}
8912
8913	/* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8914	rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8915	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8916	if (rcStrict != VINF_SUCCESS)
8917	break;
8918	if (cLoops > pVM->hm.s.cMaxResumeLoops)
8919	{
8920	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8921	rcStrict = VINF_EM_RAW_INTERRUPT;
8922	break;
8923	}
8924
8925	/*
8926	* Did the RIP change, if so, consider it a single step.
8927	* Otherwise, make sure one of the TFs gets set.
8928	*/
8929	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8930	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8931	AssertRCReturn(rc2, rc2);
8932	if ( pCtx->rip != uRipStart
8933	\|\| pCtx->cs.Sel != uCsStart)
8934	{
8935	rcStrict = VINF_EM_DBG_STEPPED;
8936	break;
8937	}
8938	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8939	}
8940
8941	/*
8942	* Clear the X86_EFL_TF if necessary.
8943	*/
8944	if (pVCpu->hm.s.fClearTrapFlag)
8945	{
8946	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8947	AssertRCReturn(rc2, rc2);
8948	pVCpu->hm.s.fClearTrapFlag = false;
8949	pCtx->eflags.Bits.u1TF = 0;
8950	}
8951	/** @todo there seems to be issues with the resume flag when the monitor trap
8952	* flag is pending without being used. Seen early in bios init when
8953	* accessing APIC page in protected mode. */
8954
8955	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8956	return rcStrict;
8957	}
8958
8959
8960	/**
8961	* Checks if any expensive dtrace probes are enabled and we should go to the
8962	* debug loop.
8963	*
8964	* @returns true if we should use debug loop, false if not.
8965	*/
8966	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
8967	{
8968	/* It's probably faster to OR the raw 32-bit counter variables together.
8969	Since the variables are in an array and the probes are next to one
8970	another (more or less), we have good locality. So, better read two three
8971	cache lines ever time and only have one conditional, than 20+ conditionals. */
8972	return ( VBOXVMM_XCPT_DE_ENABLED_RAW()
8973	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
8974	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
8975	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
8976	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
8977	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
8978	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
8979	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
8980	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
8981	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
8982	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
8983	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
8984	\| VBOXVMM_XCPT_PG_ENABLED_RAW()
8985	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
8986	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
8987	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
8988	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
8989	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
8990	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
8991	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
8992	) != 0;
8993	}
8994
8995
8996	/**
8997	* Runs the guest code using VT-x.
8998	*
8999	* @returns Strict VBox status code.
9000	* @param pVM The cross context VM structure.
9001	* @param pVCpu The cross context virtual CPU structure.
9002	* @param pCtx Pointer to the guest-CPU context.
9003	*/
9004	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9005	{
9006	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9007	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
9008	HMVMX_ASSERT_PREEMPT_SAFE();
9009
9010	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
9011
9012	VBOXSTRICTRC rcStrict;
9013	if ( !pVCpu->hm.s.fUseDebugLoop
9014	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled()) )
9015	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
9016	else
9017	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
9018
9019	if (rcStrict == VERR_EM_INTERPRETER)
9020	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9021	else if (rcStrict == VINF_EM_RESET)
9022	rcStrict = VINF_EM_TRIPLE_FAULT;
9023
9024	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
9025	if (RT_FAILURE(rc2))
9026	{
9027	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
9028	rcStrict = rc2;
9029	}
9030	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9031	return rcStrict;
9032	}
9033
9034
9035	#ifndef HMVMX_USE_FUNCTION_TABLE
9036	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9037	{
9038	# ifdef DEBUG_ramshankar
9039	# define RETURN_EXIT_CALL(a_CallExpr) \
9040	do { \
9041	/* int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); */ \
9042	VBOXSTRICTRC rcStrict = a_CallExpr; \
9043	/* HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); */ \
9044	return rcStrict; \
9045	} while (0)
9046	# else
9047	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
9048	# endif
9049	switch (rcReason)
9050	{
9051	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
9052	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
9053	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
9054	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
9055	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
9056	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
9057	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
9058	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
9059	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
9060	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
9061	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
9062	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
9063	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
9064	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
9065	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
9066	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
9067	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
9068	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
9069	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
9070	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
9071	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
9072	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
9073	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
9074	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
9075	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
9076	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
9077	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
9078	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
9079	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
9080	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
9081	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
9082	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
9083	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
9084	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
9085
9086	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
9087	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
9088	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
9089	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
9090	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
9091	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
9092	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
9093	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
9094	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
9095
9096	case VMX_EXIT_VMCLEAR:
9097	case VMX_EXIT_VMLAUNCH:
9098	case VMX_EXIT_VMPTRLD:
9099	case VMX_EXIT_VMPTRST:
9100	case VMX_EXIT_VMREAD:
9101	case VMX_EXIT_VMRESUME:
9102	case VMX_EXIT_VMWRITE:
9103	case VMX_EXIT_VMXOFF:
9104	case VMX_EXIT_VMXON:
9105	case VMX_EXIT_INVEPT:
9106	case VMX_EXIT_INVVPID:
9107	case VMX_EXIT_VMFUNC:
9108	case VMX_EXIT_XSAVES:
9109	case VMX_EXIT_XRSTORS:
9110	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9111	case VMX_EXIT_RESERVED_60:
9112	case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
9113	case VMX_EXIT_RESERVED_62:
9114	default:
9115	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9116	}
9117	}
9118	#endif /* !HMVMX_USE_FUNCTION_TABLE */
9119
9120
9121	/**
9122	* Single-stepping VM-exit filtering.
9123	*
9124	* This is preprocessing the exits and deciding whether we've gotten far enough
9125	* to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9126	* performed.
9127	*
9128	* @returns Strict VBox status code.
9129	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9130	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9131	* out-of-sync. Make sure to update the required
9132	* fields before using them.
9133	* @param pVmxTransient Pointer to the VMX-transient structure.
9134	* @param uExitReason The VM-exit reason.
9135	* @param uCsStart The CS we started executing (stepping) on.
9136	* @param uRipStart The RIP we started executing (stepping) on.
9137	*/
9138	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9139	uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9140	{
9141	switch (uExitReason)
9142	{
9143	case VMX_EXIT_XCPT_OR_NMI:
9144	{
9145	/* Check for host NMI. */
9146	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9147	AssertRCReturn(rc2, rc2);
9148	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9149	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9150	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9151	/* fall thru */
9152	}
9153
9154	case VMX_EXIT_EPT_MISCONFIG:
9155	case VMX_EXIT_TRIPLE_FAULT:
9156	case VMX_EXIT_APIC_ACCESS:
9157	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9158	case VMX_EXIT_TASK_SWITCH:
9159
9160	/* Instruction specific VM-exits: */
9161	case VMX_EXIT_IO_INSTR:
9162	case VMX_EXIT_CPUID:
9163	case VMX_EXIT_RDTSC:
9164	case VMX_EXIT_RDTSCP:
9165	case VMX_EXIT_MOV_CRX:
9166	case VMX_EXIT_MWAIT:
9167	case VMX_EXIT_MONITOR:
9168	case VMX_EXIT_RDMSR:
9169	case VMX_EXIT_WRMSR:
9170	case VMX_EXIT_MOV_DRX:
9171	case VMX_EXIT_HLT:
9172	case VMX_EXIT_INVD:
9173	case VMX_EXIT_INVLPG:
9174	case VMX_EXIT_RSM:
9175	case VMX_EXIT_PAUSE:
9176	case VMX_EXIT_XDTR_ACCESS:
9177	case VMX_EXIT_TR_ACCESS:
9178	case VMX_EXIT_WBINVD:
9179	case VMX_EXIT_XSETBV:
9180	case VMX_EXIT_RDRAND:
9181	case VMX_EXIT_INVPCID:
9182	case VMX_EXIT_GETSEC:
9183	case VMX_EXIT_RDPMC:
9184	case VMX_EXIT_VMCALL:
9185	case VMX_EXIT_VMCLEAR:
9186	case VMX_EXIT_VMLAUNCH:
9187	case VMX_EXIT_VMPTRLD:
9188	case VMX_EXIT_VMPTRST:
9189	case VMX_EXIT_VMREAD:
9190	case VMX_EXIT_VMRESUME:
9191	case VMX_EXIT_VMWRITE:
9192	case VMX_EXIT_VMXOFF:
9193	case VMX_EXIT_VMXON:
9194	case VMX_EXIT_INVEPT:
9195	case VMX_EXIT_INVVPID:
9196	case VMX_EXIT_VMFUNC:
9197	{
9198	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9199	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9200	AssertRCReturn(rc2, rc2);
9201	if ( pMixedCtx->rip != uRipStart
9202	\|\| pMixedCtx->cs.Sel != uCsStart)
9203	return VINF_EM_DBG_STEPPED;
9204	break;
9205	}
9206	}
9207
9208	/*
9209	* Normal processing.
9210	*/
9211	#ifdef HMVMX_USE_FUNCTION_TABLE
9212	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9213	#else
9214	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9215	#endif
9216	}
9217
9218
9219	#ifdef VBOX_STRICT
9220	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9221	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9222	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9223
9224	# define HMVMX_ASSERT_PREEMPT_CPUID() \
9225	do { \
9226	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9227	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9228	} while (0)
9229
9230	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9231	do { \
9232	AssertPtr(pVCpu); \
9233	AssertPtr(pMixedCtx); \
9234	AssertPtr(pVmxTransient); \
9235	Assert(pVmxTransient->fVMEntryFailed == false); \
9236	Assert(ASMIntAreEnabled()); \
9237	HMVMX_ASSERT_PREEMPT_SAFE(); \
9238	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9239	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9240	HMVMX_ASSERT_PREEMPT_SAFE(); \
9241	if (VMMR0IsLogFlushDisabled(pVCpu)) \
9242	HMVMX_ASSERT_PREEMPT_CPUID(); \
9243	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9244	} while (0)
9245
9246	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9247	do { \
9248	Log4Func(("\n")); \
9249	} while (0)
9250	#else /* nonstrict builds: */
9251	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9252	do { \
9253	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9254	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9255	} while (0)
9256	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9257	#endif
9258
9259
9260	/**
9261	* Advances the guest RIP after reading it from the VMCS.
9262	*
9263	* @returns VBox status code, no informational status codes.
9264	* @param pVCpu The cross context virtual CPU structure.
9265	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9266	* out-of-sync. Make sure to update the required fields
9267	* before using them.
9268	* @param pVmxTransient Pointer to the VMX transient structure.
9269	*
9270	* @remarks No-long-jump zone!!!
9271	*/
9272	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9273	{
9274	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9275	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9276	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9277	AssertRCReturn(rc, rc);
9278
9279	pMixedCtx->rip += pVmxTransient->cbInstr;
9280	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9281
9282	/*
9283	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9284	* pending debug exception field as it takes care of priority of events.
9285	*
9286	* See Intel spec. 32.2.1 "Debug Exceptions".
9287	*/
9288	if ( !pVCpu->hm.s.fSingleInstruction
9289	&& pMixedCtx->eflags.Bits.u1TF)
9290	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
9291
9292	return VINF_SUCCESS;
9293	}
9294
9295
9296	/**
9297	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9298	* and update error record fields accordingly.
9299	*
9300	* @return VMX_IGS_* return codes.
9301	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9302	* wrong with the guest state.
9303	*
9304	* @param pVM The cross context VM structure.
9305	* @param pVCpu The cross context virtual CPU structure.
9306	* @param pCtx Pointer to the guest-CPU state.
9307	*
9308	* @remarks This function assumes our cache of the VMCS controls
9309	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9310	*/
9311	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9312	{
9313	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9314	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9315	uError = (err); \
9316	break; \
9317	} else do { } while (0)
9318
9319	int rc;
9320	uint32_t uError = VMX_IGS_ERROR;
9321	uint32_t u32Val;
9322	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9323
9324	do
9325	{
9326	/*
9327	* CR0.
9328	*/
9329	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9330	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9331	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9332	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9333	if (fUnrestrictedGuest)
9334	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9335
9336	uint32_t u32GuestCR0;
9337	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9338	AssertRCBreak(rc);
9339	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9340	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9341	if ( !fUnrestrictedGuest
9342	&& (u32GuestCR0 & X86_CR0_PG)
9343	&& !(u32GuestCR0 & X86_CR0_PE))
9344	{
9345	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9346	}
9347
9348	/*
9349	* CR4.
9350	*/
9351	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9352	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9353
9354	uint32_t u32GuestCR4;
9355	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9356	AssertRCBreak(rc);
9357	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9358	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9359
9360	/*
9361	* IA32_DEBUGCTL MSR.
9362	*/
9363	uint64_t u64Val;
9364	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9365	AssertRCBreak(rc);
9366	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9367	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9368	{
9369	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9370	}
9371	uint64_t u64DebugCtlMsr = u64Val;
9372
9373	#ifdef VBOX_STRICT
9374	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9375	AssertRCBreak(rc);
9376	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9377	#endif
9378	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9379
9380	/*
9381	* RIP and RFLAGS.
9382	*/
9383	uint32_t u32Eflags;
9384	#if HC_ARCH_BITS == 64
9385	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9386	AssertRCBreak(rc);
9387	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9388	if ( !fLongModeGuest
9389	\|\| !pCtx->cs.Attr.n.u1Long)
9390	{
9391	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9392	}
9393	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9394	* must be identical if the "IA-32e mode guest" VM-entry
9395	* control is 1 and CS.L is 1. No check applies if the
9396	* CPU supports 64 linear-address bits. */
9397
9398	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9399	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9400	AssertRCBreak(rc);
9401	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9402	VMX_IGS_RFLAGS_RESERVED);
9403	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9404	u32Eflags = u64Val;
9405	#else
9406	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9407	AssertRCBreak(rc);
9408	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9409	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9410	#endif
9411
9412	if ( fLongModeGuest
9413	\|\| ( fUnrestrictedGuest
9414	&& !(u32GuestCR0 & X86_CR0_PE)))
9415	{
9416	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9417	}
9418
9419	uint32_t u32EntryInfo;
9420	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9421	AssertRCBreak(rc);
9422	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9423	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9424	{
9425	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9426	}
9427
9428	/*
9429	* 64-bit checks.
9430	*/
9431	#if HC_ARCH_BITS == 64
9432	if (fLongModeGuest)
9433	{
9434	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9435	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9436	}
9437
9438	if ( !fLongModeGuest
9439	&& (u32GuestCR4 & X86_CR4_PCIDE))
9440	{
9441	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9442	}
9443
9444	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9445	* 51:32 beyond the processor's physical-address width are 0. */
9446
9447	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9448	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9449	{
9450	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9451	}
9452
9453	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9454	AssertRCBreak(rc);
9455	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9456
9457	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9458	AssertRCBreak(rc);
9459	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9460	#endif
9461
9462	/*
9463	* PERF_GLOBAL MSR.
9464	*/
9465	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9466	{
9467	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9468	AssertRCBreak(rc);
9469	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9470	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9471	}
9472
9473	/*
9474	* PAT MSR.
9475	*/
9476	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9477	{
9478	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9479	AssertRCBreak(rc);
9480	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9481	for (unsigned i = 0; i < 8; i++)
9482	{
9483	uint8_t u8Val = (u64Val & 0xff);
9484	if ( u8Val != 0 /* UC */
9485	&& u8Val != 1 /* WC */
9486	&& u8Val != 4 /* WT */
9487	&& u8Val != 5 /* WP */
9488	&& u8Val != 6 /* WB */
9489	&& u8Val != 7 /* UC- */)
9490	{
9491	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9492	}
9493	u64Val >>= 8;
9494	}
9495	}
9496
9497	/*
9498	* EFER MSR.
9499	*/
9500	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9501	{
9502	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9503	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9504	AssertRCBreak(rc);
9505	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9506	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9507	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
9508	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9509	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9510	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9511	\|\| !(u32GuestCR0 & X86_CR0_PG)
9512	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9513	VMX_IGS_EFER_LMA_LME_MISMATCH);
9514	}
9515
9516	/*
9517	* Segment registers.
9518	*/
9519	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9520	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9521	if (!(u32Eflags & X86_EFL_VM))
9522	{
9523	/* CS */
9524	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9525	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9526	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9527	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9528	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9529	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9530	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9531	/* CS cannot be loaded with NULL in protected mode. */
9532	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9533	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9534	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9535	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9536	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9537	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9538	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9539	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9540	else
9541	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9542
9543	/* SS */
9544	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9545	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9546	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9547	if ( !(pCtx->cr0 & X86_CR0_PE)
9548	\|\| pCtx->cs.Attr.n.u4Type == 3)
9549	{
9550	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9551	}
9552	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9553	{
9554	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9555	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9556	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9557	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9558	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9559	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9560	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9561	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9562	}
9563
9564	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9565	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9566	{
9567	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9568	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9569	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9570	\|\| pCtx->ds.Attr.n.u4Type > 11
9571	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9572	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9573	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9574	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9575	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9576	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9577	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9578	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9579	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9580	}
9581	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9582	{
9583	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9584	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9585	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9586	\|\| pCtx->es.Attr.n.u4Type > 11
9587	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9588	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9589	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9590	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9591	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9592	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9593	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9594	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9595	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9596	}
9597	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9598	{
9599	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9600	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9601	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9602	\|\| pCtx->fs.Attr.n.u4Type > 11
9603	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9604	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9605	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9606	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9607	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9608	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9609	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9610	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9611	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9612	}
9613	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9614	{
9615	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9616	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9617	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9618	\|\| pCtx->gs.Attr.n.u4Type > 11
9619	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9620	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9621	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9622	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9623	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9624	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9625	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9626	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9627	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9628	}
9629	/* 64-bit capable CPUs. */
9630	#if HC_ARCH_BITS == 64
9631	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9632	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9633	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9634	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9635	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9636	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9637	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9638	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9639	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9640	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9641	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9642	#endif
9643	}
9644	else
9645	{
9646	/* V86 mode checks. */
9647	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9648	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9649	{
9650	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9651	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9652	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9653	}
9654	else
9655	{
9656	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9657	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9658	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9659	}
9660
9661	/* CS */
9662	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9663	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9664	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9665	/* SS */
9666	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9667	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9668	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9669	/* DS */
9670	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9671	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9672	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9673	/* ES */
9674	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9675	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9676	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9677	/* FS */
9678	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9679	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9680	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9681	/* GS */
9682	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9683	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9684	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9685	/* 64-bit capable CPUs. */
9686	#if HC_ARCH_BITS == 64
9687	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9688	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9689	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9690	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9691	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9692	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9693	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9694	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9695	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9696	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9697	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9698	#endif
9699	}
9700
9701	/*
9702	* TR.
9703	*/
9704	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9705	/* 64-bit capable CPUs. */
9706	#if HC_ARCH_BITS == 64
9707	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9708	#endif
9709	if (fLongModeGuest)
9710	{
9711	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9712	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9713	}
9714	else
9715	{
9716	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9717	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9718	VMX_IGS_TR_ATTR_TYPE_INVALID);
9719	}
9720	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9721	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9722	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9723	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9724	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9725	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9726	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9727	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9728
9729	/*
9730	* GDTR and IDTR.
9731	*/
9732	#if HC_ARCH_BITS == 64
9733	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9734	AssertRCBreak(rc);
9735	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9736
9737	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9738	AssertRCBreak(rc);
9739	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9740	#endif
9741
9742	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9743	AssertRCBreak(rc);
9744	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9745
9746	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9747	AssertRCBreak(rc);
9748	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9749
9750	/*
9751	* Guest Non-Register State.
9752	*/
9753	/* Activity State. */
9754	uint32_t u32ActivityState;
9755	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9756	AssertRCBreak(rc);
9757	HMVMX_CHECK_BREAK( !u32ActivityState
9758	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9759	VMX_IGS_ACTIVITY_STATE_INVALID);
9760	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9761	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9762	uint32_t u32IntrState;
9763	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9764	AssertRCBreak(rc);
9765	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9766	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9767	{
9768	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9769	}
9770
9771	/** @todo Activity state and injecting interrupts. Left as a todo since we
9772	* currently don't use activity states but ACTIVE. */
9773
9774	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9775	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9776
9777	/* Guest interruptibility-state. */
9778	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9779	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9780	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9781	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9782	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9783	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9784	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9785	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9786	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9787	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9788	{
9789	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9790	{
9791	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9792	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9793	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9794	}
9795	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9796	{
9797	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9798	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9799	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9800	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9801	}
9802	}
9803	/** @todo Assumes the processor is not in SMM. */
9804	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9805	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9806	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9807	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9808	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9809	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9810	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9811	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9812	{
9813	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9814	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9815	}
9816
9817	/* Pending debug exceptions. */
9818	#if HC_ARCH_BITS == 64
9819	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9820	AssertRCBreak(rc);
9821	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9822	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9823	u32Val = u64Val; /* For pending debug exceptions checks below. */
9824	#else
9825	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9826	AssertRCBreak(rc);
9827	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9828	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9829	#endif
9830
9831	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9832	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9833	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9834	{
9835	if ( (u32Eflags & X86_EFL_TF)
9836	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9837	{
9838	/* Bit 14 is PendingDebug.BS. */
9839	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9840	}
9841	if ( !(u32Eflags & X86_EFL_TF)
9842	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9843	{
9844	/* Bit 14 is PendingDebug.BS. */
9845	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9846	}
9847	}
9848
9849	/* VMCS link pointer. */
9850	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9851	AssertRCBreak(rc);
9852	if (u64Val != UINT64_C(0xffffffffffffffff))
9853	{
9854	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9855	/** @todo Bits beyond the processor's physical-address width MBZ. */
9856	/** @todo 32-bit located in memory referenced by value of this field (as a
9857	* physical address) must contain the processor's VMCS revision ID. */
9858	/** @todo SMM checks. */
9859	}
9860
9861	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9862	* not using Nested Paging? */
9863	if ( pVM->hm.s.fNestedPaging
9864	&& !fLongModeGuest
9865	&& CPUMIsGuestInPAEModeEx(pCtx))
9866	{
9867	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9868	AssertRCBreak(rc);
9869	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9870
9871	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9872	AssertRCBreak(rc);
9873	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9874
9875	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9876	AssertRCBreak(rc);
9877	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9878
9879	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9880	AssertRCBreak(rc);
9881	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9882	}
9883
9884	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9885	if (uError == VMX_IGS_ERROR)
9886	uError = VMX_IGS_REASON_NOT_FOUND;
9887	} while (0);
9888
9889	pVCpu->hm.s.u32HMError = uError;
9890	return uError;
9891
9892	#undef HMVMX_ERROR_BREAK
9893	#undef HMVMX_CHECK_BREAK
9894	}
9895
9896	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9897	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9898	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9899
9900	/** @name VM-exit handlers.
9901	* @{
9902	*/
9903
9904	/**
9905	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9906	*/
9907	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9908	{
9909	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9911	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9912	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
9913	return VINF_SUCCESS;
9914	return VINF_EM_RAW_INTERRUPT;
9915	}
9916
9917
9918	/**
9919	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9920	*/
9921	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9922	{
9923	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9924	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9925
9926	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9927	AssertRCReturn(rc, rc);
9928
9929	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9930	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9931	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9932	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9933
9934	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9935	{
9936	/*
9937	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9938	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9939	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9940	*
9941	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9942	*/
9943	VMXDispatchHostNmi();
9944	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9945	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9946	return VINF_SUCCESS;
9947	}
9948
9949	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9950	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9951	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
9952	{ /* likely */ }
9953	else
9954	{
9955	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
9956	rcStrictRc1 = VINF_SUCCESS;
9957	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9958	return rcStrictRc1;
9959	}
9960
9961	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9962	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9963	switch (uIntType)
9964	{
9965	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9966	Assert(uVector == X86_XCPT_DB);
9967	/* no break */
9968	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9969	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9970	/* no break */
9971	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9972	{
9973	switch (uVector)
9974	{
9975	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9976	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9977	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9978	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9979	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9980	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9981	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
9982	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9983	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9984	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9985	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9986	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9987	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9988	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9989	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9990	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9991	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9992	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9993	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9994	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9995	#endif
9996	default:
9997	{
9998	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9999	AssertRCReturn(rc, rc);
10000
10001	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10002	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10003	{
10004	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10005	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10006	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10007
10008	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10009	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10010	AssertRCReturn(rc, rc);
10011	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10012	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10013	0 /* GCPtrFaultAddress */);
10014	AssertRCReturn(rc, rc);
10015	}
10016	else
10017	{
10018	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10019	pVCpu->hm.s.u32HMError = uVector;
10020	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10021	}
10022	break;
10023	}
10024	}
10025	break;
10026	}
10027
10028	default:
10029	{
10030	pVCpu->hm.s.u32HMError = uExitIntInfo;
10031	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10032	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10033	break;
10034	}
10035	}
10036	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10037	return rc;
10038	}
10039
10040
10041	/**
10042	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10043	*/
10044	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10045	{
10046	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10047
10048	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10049	hmR0VmxClearIntWindowExitVmcs(pVCpu);
10050
10051	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10053	return VINF_SUCCESS;
10054	}
10055
10056
10057	/**
10058	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10059	*/
10060	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10061	{
10062	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10063	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10064	{
10065	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10066	HMVMX_RETURN_UNEXPECTED_EXIT();
10067	}
10068
10069	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10070
10071	/*
10072	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10073	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10074	*/
10075	uint32_t uIntrState = 0;
10076	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10077	AssertRCReturn(rc, rc);
10078
10079	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10080	if ( fBlockSti
10081	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10082	{
10083	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10084	}
10085
10086	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10087	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10088
10089	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10090	return VINF_SUCCESS;
10091	}
10092
10093
10094	/**
10095	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10096	*/
10097	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10098	{
10099	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10101	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10102	}
10103
10104
10105	/**
10106	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10107	*/
10108	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10109	{
10110	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10112	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10113	}
10114
10115
10116	/**
10117	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10118	*/
10119	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10120	{
10121	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10122	PVM pVM = pVCpu->CTX_SUFF(pVM);
10123	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10124	if (RT_LIKELY(rc == VINF_SUCCESS))
10125	{
10126	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10127	Assert(pVmxTransient->cbInstr == 2);
10128	}
10129	else
10130	{
10131	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10132	rc = VERR_EM_INTERPRETER;
10133	}
10134	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10135	return rc;
10136	}
10137
10138
10139	/**
10140	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10141	*/
10142	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10143	{
10144	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10145	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10146	AssertRCReturn(rc, rc);
10147
10148	if (pMixedCtx->cr4 & X86_CR4_SMXE)
10149	return VINF_EM_RAW_EMULATE_INSTR;
10150
10151	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10152	HMVMX_RETURN_UNEXPECTED_EXIT();
10153	}
10154
10155
10156	/**
10157	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10158	*/
10159	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10160	{
10161	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10162	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10163	AssertRCReturn(rc, rc);
10164
10165	PVM pVM = pVCpu->CTX_SUFF(pVM);
10166	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10167	if (RT_LIKELY(rc == VINF_SUCCESS))
10168	{
10169	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10170	Assert(pVmxTransient->cbInstr == 2);
10171	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10172	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10173	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10174	}
10175	else
10176	rc = VERR_EM_INTERPRETER;
10177	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10178	return rc;
10179	}
10180
10181
10182	/**
10183	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10184	*/
10185	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10186	{
10187	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10188	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10189	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10190	AssertRCReturn(rc, rc);
10191
10192	PVM pVM = pVCpu->CTX_SUFF(pVM);
10193	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10194	if (RT_SUCCESS(rc))
10195	{
10196	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10197	Assert(pVmxTransient->cbInstr == 3);
10198	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10199	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10200	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10201	}
10202	else
10203	{
10204	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10205	rc = VERR_EM_INTERPRETER;
10206	}
10207	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10208	return rc;
10209	}
10210
10211
10212	/**
10213	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10214	*/
10215	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10216	{
10217	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10218	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10219	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10220	AssertRCReturn(rc, rc);
10221
10222	PVM pVM = pVCpu->CTX_SUFF(pVM);
10223	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10224	if (RT_LIKELY(rc == VINF_SUCCESS))
10225	{
10226	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10227	Assert(pVmxTransient->cbInstr == 2);
10228	}
10229	else
10230	{
10231	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10232	rc = VERR_EM_INTERPRETER;
10233	}
10234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10235	return rc;
10236	}
10237
10238
10239	/**
10240	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10241	*/
10242	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10243	{
10244	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10246
10247	if (pVCpu->hm.s.fHypercallsEnabled)
10248	{
10249	#if 0
10250	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10251	AssertRCReturn(rc, rc);
10252	#else
10253	/* Aggressive state sync. for now. */
10254	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10255	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
10256	#endif
10257	rc \|= hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10258	AssertRCReturn(rc, rc);
10259
10260	/** @todo pre-increment RIP before hypercall will break when we have to implement
10261	* continuing hypercalls (e.g. Hyper-V). */
10262	/** @todo r=bird: GIMHypercall will probably have to be able to return
10263	* informational status codes, so it should be made VBOXSTRICTRC. Not
10264	* doing that now because the status code handling isn't clean (i.e.
10265	* if you use RT_SUCCESS(rc) on the result of something, you don't
10266	* return rc in the success case, you return VINF_SUCCESS). */
10267	rc = GIMHypercall(pVCpu, pMixedCtx);
10268	/* If the hypercall changes anything other than guest general-purpose registers,
10269	we would need to reload the guest changed bits here before VM-entry. */
10270	return rc;
10271	}
10272
10273	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
10274	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10275	return VINF_SUCCESS;
10276	}
10277
10278
10279	/**
10280	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10281	*/
10282	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10283	{
10284	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10285	PVM pVM = pVCpu->CTX_SUFF(pVM);
10286	Assert(!pVM->hm.s.fNestedPaging);
10287
10288	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10289	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10290	AssertRCReturn(rc, rc);
10291
10292	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10293	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10294	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10295	else
10296	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10297	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
10298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10299	return rcStrict;
10300	}
10301
10302
10303	/**
10304	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10305	*/
10306	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10307	{
10308	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10309	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10310	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10311	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10312	AssertRCReturn(rc, rc);
10313
10314	PVM pVM = pVCpu->CTX_SUFF(pVM);
10315	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10316	if (RT_LIKELY(rc == VINF_SUCCESS))
10317	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10318	else
10319	{
10320	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10321	rc = VERR_EM_INTERPRETER;
10322	}
10323	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10324	return rc;
10325	}
10326
10327
10328	/**
10329	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10330	*/
10331	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10332	{
10333	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10334	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10335	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10336	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10337	AssertRCReturn(rc, rc);
10338
10339	PVM pVM = pVCpu->CTX_SUFF(pVM);
10340	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10341	rc = VBOXSTRICTRC_VAL(rc2);
10342	if (RT_LIKELY( rc == VINF_SUCCESS
10343	\|\| rc == VINF_EM_HALT))
10344	{
10345	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10346	AssertRCReturn(rc3, rc3);
10347
10348	if ( rc == VINF_EM_HALT
10349	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10350	{
10351	rc = VINF_SUCCESS;
10352	}
10353	}
10354	else
10355	{
10356	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10357	rc = VERR_EM_INTERPRETER;
10358	}
10359	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
10360	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10361	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10362	return rc;
10363	}
10364
10365
10366	/**
10367	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10368	*/
10369	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10370	{
10371	/*
10372	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10373	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10374	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
10375	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10376	*/
10377	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10378	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10379	HMVMX_RETURN_UNEXPECTED_EXIT();
10380	}
10381
10382
10383	/**
10384	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10385	*/
10386	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10387	{
10388	/*
10389	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10390	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10391	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
10392	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10393	*/
10394	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10395	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10396	HMVMX_RETURN_UNEXPECTED_EXIT();
10397	}
10398
10399
10400	/**
10401	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10402	*/
10403	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10404	{
10405	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10406	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10407	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10408	HMVMX_RETURN_UNEXPECTED_EXIT();
10409	}
10410
10411
10412	/**
10413	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10414	*/
10415	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10416	{
10417	/*
10418	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10419	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10420	* See Intel spec. 25.3 "Other Causes of VM-exits".
10421	*/
10422	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10423	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10424	HMVMX_RETURN_UNEXPECTED_EXIT();
10425	}
10426
10427
10428	/**
10429	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10430	* VM-exit.
10431	*/
10432	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10433	{
10434	/*
10435	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10436	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10437	*
10438	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10439	* See Intel spec. "23.8 Restrictions on VMX operation".
10440	*/
10441	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10442	return VINF_SUCCESS;
10443	}
10444
10445
10446	/**
10447	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10448	* VM-exit.
10449	*/
10450	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10451	{
10452	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10453	return VINF_EM_RESET;
10454	}
10455
10456
10457	/**
10458	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10459	*/
10460	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10461	{
10462	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10463	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10464	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10465	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10466	AssertRCReturn(rc, rc);
10467
10468	pMixedCtx->rip++;
10469	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10470	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10471	rc = VINF_SUCCESS;
10472	else
10473	rc = VINF_EM_HALT;
10474
10475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10476	if (rc != VINF_SUCCESS)
10477	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
10478	return rc;
10479	}
10480
10481
10482	/**
10483	* VM-exit handler for instructions that result in a \#UD exception delivered to
10484	* the guest.
10485	*/
10486	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10487	{
10488	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10489	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10490	return VINF_SUCCESS;
10491	}
10492
10493
10494	/**
10495	* VM-exit handler for expiry of the VMX preemption timer.
10496	*/
10497	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10498	{
10499	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10500
10501	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10502	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10503
10504	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10505	PVM pVM = pVCpu->CTX_SUFF(pVM);
10506	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10507	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10508	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10509	}
10510
10511
10512	/**
10513	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10514	*/
10515	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10516	{
10517	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10518
10519	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10520	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
10521	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10522	AssertRCReturn(rc, rc);
10523
10524	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
10525	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10526
10527	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
10528
10529	return rcStrict;
10530	}
10531
10532
10533	/**
10534	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10535	*/
10536	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10537	{
10538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10539
10540	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
10541	/** @todo implement EMInterpretInvpcid() */
10542	return VERR_EM_INTERPRETER;
10543	}
10544
10545
10546	/**
10547	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10548	* Error VM-exit.
10549	*/
10550	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10551	{
10552	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10553	AssertRCReturn(rc, rc);
10554
10555	rc = hmR0VmxCheckVmcsCtls(pVCpu);
10556	AssertRCReturn(rc, rc);
10557
10558	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10559	NOREF(uInvalidReason);
10560
10561	#ifdef VBOX_STRICT
10562	uint32_t uIntrState;
10563	RTHCUINTREG uHCReg;
10564	uint64_t u64Val;
10565	uint32_t u32Val;
10566
10567	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10568	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10569	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10570	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10571	AssertRCReturn(rc, rc);
10572
10573	Log4(("uInvalidReason %u\n", uInvalidReason));
10574	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10575	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10576	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10577	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10578
10579	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10580	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10581	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10582	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10583	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10584	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10585	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10586	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10587	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10588	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10589	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10590	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10591	#else
10592	NOREF(pVmxTransient);
10593	#endif
10594
10595	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10596	return VERR_VMX_INVALID_GUEST_STATE;
10597	}
10598
10599
10600	/**
10601	* VM-exit handler for VM-entry failure due to an MSR-load
10602	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10603	*/
10604	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10605	{
10606	NOREF(pVmxTransient);
10607	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10608	HMVMX_RETURN_UNEXPECTED_EXIT();
10609	}
10610
10611
10612	/**
10613	* VM-exit handler for VM-entry failure due to a machine-check event
10614	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10615	*/
10616	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10617	{
10618	NOREF(pVmxTransient);
10619	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10620	HMVMX_RETURN_UNEXPECTED_EXIT();
10621	}
10622
10623
10624	/**
10625	* VM-exit handler for all undefined reasons. Should never ever happen.. in
10626	* theory.
10627	*/
10628	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10629	{
10630	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10631	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10632	return VERR_VMX_UNDEFINED_EXIT_CODE;
10633	}
10634
10635
10636	/**
10637	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10638	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10639	* Conditional VM-exit.
10640	*/
10641	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10642	{
10643	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10644
10645	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10647	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10648	return VERR_EM_INTERPRETER;
10649	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10650	HMVMX_RETURN_UNEXPECTED_EXIT();
10651	}
10652
10653
10654	/**
10655	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10656	*/
10657	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10658	{
10659	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10660
10661	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10663	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10664	return VERR_EM_INTERPRETER;
10665	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10666	HMVMX_RETURN_UNEXPECTED_EXIT();
10667	}
10668
10669
10670	/**
10671	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10672	*/
10673	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10674	{
10675	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10676
10677	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10678	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10679	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10680	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10681	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10682	{
10683	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10684	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10685	}
10686	AssertRCReturn(rc, rc);
10687	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10688
10689	#ifdef VBOX_STRICT
10690	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10691	{
10692	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10693	&& pMixedCtx->ecx != MSR_K6_EFER)
10694	{
10695	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10696	pMixedCtx->ecx));
10697	HMVMX_RETURN_UNEXPECTED_EXIT();
10698	}
10699	# if HC_ARCH_BITS == 64
10700	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10701	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10702	{
10703	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10704	HMVMX_RETURN_UNEXPECTED_EXIT();
10705	}
10706	# endif
10707	}
10708	#endif
10709
10710	PVM pVM = pVCpu->CTX_SUFF(pVM);
10711	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10712	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
10713	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10714	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10715	if (RT_SUCCESS(rc))
10716	{
10717	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10718	Assert(pVmxTransient->cbInstr == 2);
10719	}
10720	return rc;
10721	}
10722
10723
10724	/**
10725	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10726	*/
10727	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10728	{
10729	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10730	PVM pVM = pVCpu->CTX_SUFF(pVM);
10731	int rc = VINF_SUCCESS;
10732
10733	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10734	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10735	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10736	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10737	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10738	{
10739	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10740	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10741	}
10742	AssertRCReturn(rc, rc);
10743	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10744
10745	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10746	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10747	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10748
10749	if (RT_SUCCESS(rc))
10750	{
10751	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10752
10753	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10754	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10755	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10756	{
10757	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10758	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10759	EMInterpretWrmsr() changes it. */
10760	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10761	}
10762	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10763	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10764	else if (pMixedCtx->ecx == MSR_K6_EFER)
10765	{
10766	/*
10767	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10768	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10769	* the other bits as well, SCE and NXE. See @bugref{7368}.
10770	*/
10771	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
10772	}
10773
10774	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10775	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10776	{
10777	switch (pMixedCtx->ecx)
10778	{
10779	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10780	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10781	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10782	case MSR_K8_FS_BASE: /* no break */
10783	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10784	case MSR_K6_EFER: /* already handled above */ break;
10785	default:
10786	{
10787	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10788	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10789	#if HC_ARCH_BITS == 64
10790	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10791	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10792	#endif
10793	break;
10794	}
10795	}
10796	}
10797	#ifdef VBOX_STRICT
10798	else
10799	{
10800	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10801	switch (pMixedCtx->ecx)
10802	{
10803	case MSR_IA32_SYSENTER_CS:
10804	case MSR_IA32_SYSENTER_EIP:
10805	case MSR_IA32_SYSENTER_ESP:
10806	case MSR_K8_FS_BASE:
10807	case MSR_K8_GS_BASE:
10808	{
10809	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10810	HMVMX_RETURN_UNEXPECTED_EXIT();
10811	}
10812
10813	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10814	default:
10815	{
10816	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10817	{
10818	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10819	if (pMixedCtx->ecx != MSR_K6_EFER)
10820	{
10821	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10822	pMixedCtx->ecx));
10823	HMVMX_RETURN_UNEXPECTED_EXIT();
10824	}
10825	}
10826
10827	#if HC_ARCH_BITS == 64
10828	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10829	{
10830	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10831	HMVMX_RETURN_UNEXPECTED_EXIT();
10832	}
10833	#endif
10834	break;
10835	}
10836	}
10837	}
10838	#endif /* VBOX_STRICT */
10839	}
10840	return rc;
10841	}
10842
10843
10844	/**
10845	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10846	*/
10847	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10848	{
10849	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10850
10851	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10852	return VINF_EM_RAW_INTERRUPT;
10853	}
10854
10855
10856	/**
10857	* VM-exit handler for when the TPR value is lowered below the specified
10858	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10859	*/
10860	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10861	{
10862	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10863	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10864
10865	/*
10866	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10867	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10868	* resume guest execution.
10869	*/
10870	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10871	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10872	return VINF_SUCCESS;
10873	}
10874
10875
10876	/**
10877	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10878	* VM-exit.
10879	*
10880	* @retval VINF_SUCCESS when guest execution can continue.
10881	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10882	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10883	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10884	* interpreter.
10885	*/
10886	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10887	{
10888	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10889	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10890	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10891	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10892	AssertRCReturn(rc, rc);
10893
10894	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10895	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10896	PVM pVM = pVCpu->CTX_SUFF(pVM);
10897	VBOXSTRICTRC rcStrict;
10898	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
10899	switch (uAccessType)
10900	{
10901	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10902	{
10903	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10904	AssertRCReturn(rc, rc);
10905
10906	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
10907	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10908	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10909	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
10910	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10911	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10912	{
10913	case 0: /* CR0 */
10914	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10915	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
10916	break;
10917	case 2: /* CR2 */
10918	/* Nothing to do here, CR2 it's not part of the VMCS. */
10919	break;
10920	case 3: /* CR3 */
10921	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10922	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10923	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
10924	break;
10925	case 4: /* CR4 */
10926	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10927	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
10928	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
10929	break;
10930	case 8: /* CR8 */
10931	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10932	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
10933	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10934	break;
10935	default:
10936	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10937	break;
10938	}
10939
10940	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10941	break;
10942	}
10943
10944	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10945	{
10946	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10947	AssertRCReturn(rc, rc);
10948
10949	Assert( !pVM->hm.s.fNestedPaging
10950	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10951	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10952
10953	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10954	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10955	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10956
10957	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
10958	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10959	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10960	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10961	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10962	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10963	VBOXSTRICTRC_VAL(rcStrict)));
10964	break;
10965	}
10966
10967	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10968	{
10969	AssertRCReturn(rc, rc);
10970	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
10971	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10972	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10973	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10974	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
10975	break;
10976	}
10977
10978	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10979	{
10980	AssertRCReturn(rc, rc);
10981	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
10982	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10983	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
10984	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10985	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10986	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
10987	break;
10988	}
10989
10990	default:
10991	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
10992	VERR_VMX_UNEXPECTED_EXCEPTION);
10993	}
10994
10995	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10996	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10997	NOREF(pVM);
10998	return rcStrict;
10999	}
11000
11001
11002	/**
11003	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
11004	* VM-exit.
11005	*/
11006	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11007	{
11008	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11009	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
11010
11011	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11012	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11013	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11014	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11015	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11016	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11017	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11018	AssertRCReturn(rc2, rc2);
11019
11020	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11021	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11022	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11023	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11024	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11025	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11026	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11027	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
11028	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11029
11030	/* I/O operation lookup arrays. */
11031	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11032	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11033
11034	VBOXSTRICTRC rcStrict;
11035	uint32_t const cbValue = s_aIOSizes[uIOWidth];
11036	uint32_t const cbInstr = pVmxTransient->cbInstr;
11037	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11038	PVM pVM = pVCpu->CTX_SUFF(pVM);
11039	if (fIOString)
11040	{
11041	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
11042	See @bugref{5752#c158}. Should work now. */
11043	/*
11044	* INS/OUTS - I/O String instruction.
11045	*
11046	* Use instruction-information if available, otherwise fall back on
11047	* interpreting the instruction.
11048	*/
11049	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
11050	fIOWrite ? 'w' : 'r'));
11051	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11052	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11053	{
11054	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11055	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11056	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11057	AssertRCReturn(rc2, rc2);
11058	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11059	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11060	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11061	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11062	if (fIOWrite)
11063	{
11064	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11065	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11066	}
11067	else
11068	{
11069	/*
11070	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11071	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11072	* See Intel Instruction spec. for "INS".
11073	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11074	*/
11075	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11076	}
11077	}
11078	else
11079	{
11080	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11081	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11082	AssertRCReturn(rc2, rc2);
11083	rcStrict = IEMExecOne(pVCpu);
11084	}
11085	/** @todo IEM needs to be setting these flags somehow. */
11086	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11087	fUpdateRipAlready = true;
11088	#else
11089	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11090	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11091	if (RT_SUCCESS(rcStrict))
11092	{
11093	if (fIOWrite)
11094	{
11095	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11096	(DISCPUMODE)pDis->uAddrMode, cbValue);
11097	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11098	}
11099	else
11100	{
11101	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11102	(DISCPUMODE)pDis->uAddrMode, cbValue);
11103	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11104	}
11105	}
11106	else
11107	{
11108	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
11109	pMixedCtx->rip));
11110	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11111	}
11112	#endif
11113	}
11114	else
11115	{
11116	/*
11117	* IN/OUT - I/O instruction.
11118	*/
11119	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11120	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11121	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11122	if (fIOWrite)
11123	{
11124	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11125	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11126	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11127	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11128	}
11129	else
11130	{
11131	uint32_t u32Result = 0;
11132	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11133	if (IOM_SUCCESS(rcStrict))
11134	{
11135	/* Save result of I/O IN instr. in AL/AX/EAX. */
11136	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
11137	}
11138	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11139	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11141	}
11142	}
11143
11144	if (IOM_SUCCESS(rcStrict))
11145	{
11146	if (!fUpdateRipAlready)
11147	{
11148	pMixedCtx->rip += cbInstr;
11149	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11150	}
11151
11152	/*
11153	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11154	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11155	*/
11156	if (fIOString)
11157	{
11158	/** @todo Single-step for INS/OUTS with REP prefix? */
11159	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11160	}
11161	else if ( !fDbgStepping
11162	&& fGstStepping)
11163	{
11164	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11165	}
11166
11167	/*
11168	* If any I/O breakpoints are armed, we need to check if one triggered
11169	* and take appropriate action.
11170	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11171	*/
11172	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11173	AssertRCReturn(rc2, rc2);
11174
11175	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11176	* execution engines about whether hyper BPs and such are pending. */
11177	uint32_t const uDr7 = pMixedCtx->dr[7];
11178	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11179	&& X86_DR7_ANY_RW_IO(uDr7)
11180	&& (pMixedCtx->cr4 & X86_CR4_DE))
11181	\|\| DBGFBpIsHwIoArmed(pVM)))
11182	{
11183	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11184
11185	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11186	VMMRZCallRing3Disable(pVCpu);
11187	HM_DISABLE_PREEMPT();
11188
11189	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11190
11191	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11192	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11193	{
11194	/* Raise #DB. */
11195	if (fIsGuestDbgActive)
11196	ASMSetDR6(pMixedCtx->dr[6]);
11197	if (pMixedCtx->dr[7] != uDr7)
11198	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11199
11200	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11201	}
11202	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11203	else if ( rcStrict2 != VINF_SUCCESS
11204	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
11205	rcStrict = rcStrict2;
11206
11207	HM_RESTORE_PREEMPT();
11208	VMMRZCallRing3Enable(pVCpu);
11209	}
11210	}
11211
11212	#ifdef VBOX_STRICT
11213	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11214	Assert(!fIOWrite);
11215	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11216	Assert(fIOWrite);
11217	else
11218	{
11219	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11220	* statuses, that the VMM device and some others may return. See
11221	* IOM_SUCCESS() for guidance. */
11222	AssertMsg( RT_FAILURE(rcStrict)
11223	\|\| rcStrict == VINF_SUCCESS
11224	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
11225	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
11226	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
11227	\|\| rcStrict == VINF_EM_RAW_TO_R3
11228	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11229	#endif
11230	}
11231	#endif
11232
11233	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11234	return rcStrict;
11235	}
11236
11237
11238	/**
11239	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11240	* VM-exit.
11241	*/
11242	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11243	{
11244	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11245
11246	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
11247	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11248	AssertRCReturn(rc, rc);
11249	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11250	{
11251	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11252	AssertRCReturn(rc, rc);
11253	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11254	{
11255	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11256
11257	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11258	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11259
11260	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11261	Assert(!pVCpu->hm.s.Event.fPending);
11262	pVCpu->hm.s.Event.fPending = true;
11263	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11264	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11265	AssertRCReturn(rc, rc);
11266	if (fErrorCodeValid)
11267	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11268	else
11269	pVCpu->hm.s.Event.u32ErrCode = 0;
11270	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11271	&& uVector == X86_XCPT_PF)
11272	{
11273	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11274	}
11275
11276	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11278	return VINF_EM_RAW_INJECT_TRPM_EVENT;
11279	}
11280	}
11281
11282	/** @todo Emulate task switch someday, currently just going back to ring-3 for
11283	* emulation. */
11284	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11285	return VERR_EM_INTERPRETER;
11286	}
11287
11288
11289	/**
11290	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11291	*/
11292	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11293	{
11294	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11295	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11296	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11297	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11298	AssertRCReturn(rc, rc);
11299	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11300	return VINF_EM_DBG_STEPPED;
11301	}
11302
11303
11304	/**
11305	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11306	*/
11307	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11308	{
11309	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11310
11311	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11312	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11313	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
11314	{ /* likely */ }
11315	else
11316	{
11317	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
11318	rcStrict1 = VINF_SUCCESS;
11319	return rcStrict1;
11320	}
11321
11322	#if 0
11323	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11324	* just sync the whole thing. */
11325	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11326	#else
11327	/* Aggressive state sync. for now. */
11328	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11329	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11330	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11331	#endif
11332	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11333	AssertRCReturn(rc, rc);
11334
11335	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11336	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11337	VBOXSTRICTRC rcStrict2;
11338	switch (uAccessType)
11339	{
11340	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11341	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11342	{
11343	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11344	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11345	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11346
11347	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11348	GCPhys &= PAGE_BASE_GC_MASK;
11349	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11350	PVM pVM = pVCpu->CTX_SUFF(pVM);
11351	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11352	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11353
11354	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
11355	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11356	CPUMCTX2CORE(pMixedCtx), GCPhys);
11357	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
11358	if ( rcStrict2 == VINF_SUCCESS
11359	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
11360	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
11361	{
11362	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11363	\| HM_CHANGED_GUEST_RSP
11364	\| HM_CHANGED_GUEST_RFLAGS
11365	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11366	rcStrict2 = VINF_SUCCESS;
11367	}
11368	break;
11369	}
11370
11371	default:
11372	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11373	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
11374	break;
11375	}
11376
11377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11378	if (rcStrict2 != VINF_SUCCESS)
11379	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
11380	return rcStrict2;
11381	}
11382
11383
11384	/**
11385	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11386	* VM-exit.
11387	*/
11388	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11389	{
11390	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11391
11392	/* We should -not- get this VM-exit if the guest's debug registers were active. */
11393	if (pVmxTransient->fWasGuestDebugStateActive)
11394	{
11395	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11396	HMVMX_RETURN_UNEXPECTED_EXIT();
11397	}
11398
11399	if ( !pVCpu->hm.s.fSingleInstruction
11400	&& !pVmxTransient->fWasHyperDebugStateActive)
11401	{
11402	Assert(!DBGFIsStepping(pVCpu));
11403	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
11404
11405	/* Don't intercept MOV DRx any more. */
11406	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11407	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11408	AssertRCReturn(rc, rc);
11409
11410	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11411	VMMRZCallRing3Disable(pVCpu);
11412	HM_DISABLE_PREEMPT();
11413
11414	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11415	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11416	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
11417
11418	HM_RESTORE_PREEMPT();
11419	VMMRZCallRing3Enable(pVCpu);
11420
11421	#ifdef VBOX_WITH_STATISTICS
11422	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11423	AssertRCReturn(rc, rc);
11424	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11425	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11426	else
11427	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11428	#endif
11429	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11430	return VINF_SUCCESS;
11431	}
11432
11433	/*
11434	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11435	* Update the segment registers and DR7 from the CPU.
11436	*/
11437	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11438	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11439	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11440	AssertRCReturn(rc, rc);
11441	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11442
11443	PVM pVM = pVCpu->CTX_SUFF(pVM);
11444	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11445	{
11446	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11447	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11448	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11449	if (RT_SUCCESS(rc))
11450	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11451	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11452	}
11453	else
11454	{
11455	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11456	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11457	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11458	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11459	}
11460
11461	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
11462	if (RT_SUCCESS(rc))
11463	{
11464	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11465	AssertRCReturn(rc2, rc2);
11466	return VINF_SUCCESS;
11467	}
11468	return rc;
11469	}
11470
11471
11472	/**
11473	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11474	* Conditional VM-exit.
11475	*/
11476	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11477	{
11478	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11479	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11480
11481	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11482	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11483	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
11484	{ /* likely */ }
11485	else
11486	{
11487	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
11488	rcStrict1 = VINF_SUCCESS;
11489	return rcStrict1;
11490	}
11491
11492	RTGCPHYS GCPhys = 0;
11493	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11494
11495	#if 0
11496	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11497	#else
11498	/* Aggressive state sync. for now. */
11499	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11500	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11501	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11502	#endif
11503	AssertRCReturn(rc, rc);
11504
11505	/*
11506	* If we succeed, resume guest execution.
11507	* If we fail in interpreting the instruction because we couldn't get the guest physical address
11508	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11509	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11510	* weird case. See @bugref{6043}.
11511	*/
11512	PVM pVM = pVCpu->CTX_SUFF(pVM);
11513	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11514	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
11515	if ( rcStrict2 == VINF_SUCCESS
11516	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
11517	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
11518	{
11519	/* Successfully handled MMIO operation. */
11520	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11521	\| HM_CHANGED_GUEST_RSP
11522	\| HM_CHANGED_GUEST_RFLAGS
11523	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11524	return VINF_SUCCESS;
11525	}
11526	return rcStrict2;
11527	}
11528
11529
11530	/**
11531	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11532	* VM-exit.
11533	*/
11534	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535	{
11536	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11537	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11538
11539	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11540	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11541	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
11542	{ /* likely */ }
11543	else
11544	{
11545	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
11546	rcStrict1 = VINF_SUCCESS;
11547	return rcStrict1;
11548	}
11549
11550	RTGCPHYS GCPhys = 0;
11551	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11552	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11553	#if 0
11554	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11555	#else
11556	/* Aggressive state sync. for now. */
11557	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11558	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11559	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11560	#endif
11561	AssertRCReturn(rc, rc);
11562
11563	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11564	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11565
11566	RTGCUINT uErrorCode = 0;
11567	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11568	uErrorCode \|= X86_TRAP_PF_ID;
11569	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11570	uErrorCode \|= X86_TRAP_PF_RW;
11571	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11572	uErrorCode \|= X86_TRAP_PF_P;
11573
11574	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11575
11576	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11577	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11578
11579	/* Handle the pagefault trap for the nested shadow table. */
11580	PVM pVM = pVCpu->CTX_SUFF(pVM);
11581	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11582	TRPMResetTrap(pVCpu);
11583
11584	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11585	if ( rcStrict2 == VINF_SUCCESS
11586	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
11587	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
11588	{
11589	/* Successfully synced our nested page tables. */
11590	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11591	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11592	\| HM_CHANGED_GUEST_RSP
11593	\| HM_CHANGED_GUEST_RFLAGS);
11594	return VINF_SUCCESS;
11595	}
11596
11597	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
11598	return rcStrict2;
11599	}
11600
11601	/** @} */
11602
11603	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11604	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11605	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11606
11607	/** @name VM-exit exception handlers.
11608	* @{
11609	*/
11610
11611	/**
11612	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
11613	*/
11614	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11615	{
11616	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11617	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11618
11619	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11620	AssertRCReturn(rc, rc);
11621
11622	if (!(pMixedCtx->cr0 & X86_CR0_NE))
11623	{
11624	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11625	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11626
11627	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11628	* provides VM-exit instruction length. If this causes problem later,
11629	* disassemble the instruction like it's done on AMD-V. */
11630	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11631	AssertRCReturn(rc2, rc2);
11632	return rc;
11633	}
11634
11635	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11636	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11637	return rc;
11638	}
11639
11640
11641	/**
11642	* VM-exit exception handler for \#BP (Breakpoint exception).
11643	*/
11644	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11645	{
11646	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11647	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11648
11649	/** @todo Try optimize this by not saving the entire guest state unless
11650	* really needed. */
11651	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11652	AssertRCReturn(rc, rc);
11653
11654	PVM pVM = pVCpu->CTX_SUFF(pVM);
11655	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11656	if (rc == VINF_EM_RAW_GUEST_TRAP)
11657	{
11658	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11659	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11660	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11661	AssertRCReturn(rc, rc);
11662
11663	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11664	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11665	}
11666
11667	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
11668	return rc;
11669	}
11670
11671
11672	/**
11673	* VM-exit exception handler for \#AC (alignment check exception).
11674	*/
11675	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11676	{
11677	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11678
11679	/*
11680	* Re-inject it. We'll detect any nesting before getting here.
11681	*/
11682	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11683	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11684	AssertRCReturn(rc, rc);
11685	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11686
11687	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11688	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11689	return VINF_SUCCESS;
11690	}
11691
11692
11693	/**
11694	* VM-exit exception handler for \#DB (Debug exception).
11695	*/
11696	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11697	{
11698	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11700	Log6(("XcptDB\n"));
11701
11702	/*
11703	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11704	* for processing.
11705	*/
11706	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11707	AssertRCReturn(rc, rc);
11708
11709	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11710	uint64_t uDR6 = X86_DR6_INIT_VAL;
11711	uDR6 \|= ( pVmxTransient->uExitQualification
11712	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
11713
11714	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11715	if (rc == VINF_EM_RAW_GUEST_TRAP)
11716	{
11717	/*
11718	* The exception was for the guest. Update DR6, DR7.GD and
11719	* IA32_DEBUGCTL.LBR before forwarding it.
11720	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11721	*/
11722	VMMRZCallRing3Disable(pVCpu);
11723	HM_DISABLE_PREEMPT();
11724
11725	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11726	pMixedCtx->dr[6] \|= uDR6;
11727	if (CPUMIsGuestDebugStateActive(pVCpu))
11728	ASMSetDR6(pMixedCtx->dr[6]);
11729
11730	HM_RESTORE_PREEMPT();
11731	VMMRZCallRing3Enable(pVCpu);
11732
11733	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11734	AssertRCReturn(rc, rc);
11735
11736	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11737	pMixedCtx->dr[7] &= ~X86_DR7_GD;
11738
11739	/* Paranoia. */
11740	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11741	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
11742
11743	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11744	AssertRCReturn(rc, rc);
11745
11746	/*
11747	* Raise #DB in the guest.
11748	*
11749	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11750	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11751	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11752	*
11753	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11754	*/
11755	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11756	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11757	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11758	AssertRCReturn(rc, rc);
11759	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11760	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11761	return VINF_SUCCESS;
11762	}
11763
11764	/*
11765	* Not a guest trap, must be a hypervisor related debug event then.
11766	* Update DR6 in case someone is interested in it.
11767	*/
11768	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11769	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11770	CPUMSetHyperDR6(pVCpu, uDR6);
11771
11772	return rc;
11773	}
11774
11775
11776	/**
11777	* VM-exit exception handler for \#NM (Device-not-available exception: floating
11778	* point exception).
11779	*/
11780	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11781	{
11782	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11783
11784	/* We require CR0 and EFER. EFER is always up-to-date. */
11785	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11786	AssertRCReturn(rc, rc);
11787
11788	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11789	VMMRZCallRing3Disable(pVCpu);
11790	HM_DISABLE_PREEMPT();
11791
11792	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11793	if (pVmxTransient->fWasGuestFPUStateActive)
11794	{
11795	rc = VINF_EM_RAW_GUEST_TRAP;
11796	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11797	}
11798	else
11799	{
11800	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11801	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11802	#endif
11803	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11804	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11805	}
11806
11807	HM_RESTORE_PREEMPT();
11808	VMMRZCallRing3Enable(pVCpu);
11809
11810	if (rc == VINF_SUCCESS)
11811	{
11812	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11813	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11814	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11815	pVCpu->hm.s.fPreloadGuestFpu = true;
11816	}
11817	else
11818	{
11819	/* Forward #NM to the guest. */
11820	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11821	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11822	AssertRCReturn(rc, rc);
11823	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11824	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11825	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11826	}
11827
11828	return VINF_SUCCESS;
11829	}
11830
11831
11832	/**
11833	* VM-exit exception handler for \#GP (General-protection exception).
11834	*
11835	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11836	*/
11837	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11838	{
11839	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11840	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11841
11842	int rc = VERR_INTERNAL_ERROR_5;
11843	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11844	{
11845	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11846	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11847	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11848	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11849	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11850	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11851	AssertRCReturn(rc, rc);
11852	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11853	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11854	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11855	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11856	return rc;
11857	#else
11858	/* We don't intercept #GP. */
11859	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11860	NOREF(pVmxTransient);
11861	return VERR_VMX_UNEXPECTED_EXCEPTION;
11862	#endif
11863	}
11864
11865	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11866	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11867
11868	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11869	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11870	AssertRCReturn(rc, rc);
11871
11872	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11873	uint32_t cbOp = 0;
11874	PVM pVM = pVCpu->CTX_SUFF(pVM);
11875	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
11876	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11877	if (RT_SUCCESS(rc))
11878	{
11879	rc = VINF_SUCCESS;
11880	Assert(cbOp == pDis->cbInstr);
11881	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11882	switch (pDis->pCurInstr->uOpcode)
11883	{
11884	case OP_CLI:
11885	{
11886	pMixedCtx->eflags.Bits.u1IF = 0;
11887	pMixedCtx->eflags.Bits.u1RF = 0;
11888	pMixedCtx->rip += pDis->cbInstr;
11889	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11890	if ( !fDbgStepping
11891	&& pMixedCtx->eflags.Bits.u1TF)
11892	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11893	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11894	break;
11895	}
11896
11897	case OP_STI:
11898	{
11899	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11900	pMixedCtx->eflags.Bits.u1IF = 1;
11901	pMixedCtx->eflags.Bits.u1RF = 0;
11902	pMixedCtx->rip += pDis->cbInstr;
11903	if (!fOldIF)
11904	{
11905	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11906	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11907	}
11908	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11909	if ( !fDbgStepping
11910	&& pMixedCtx->eflags.Bits.u1TF)
11911	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11912	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11913	break;
11914	}
11915
11916	case OP_HLT:
11917	{
11918	rc = VINF_EM_HALT;
11919	pMixedCtx->rip += pDis->cbInstr;
11920	pMixedCtx->eflags.Bits.u1RF = 0;
11921	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11922	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11923	break;
11924	}
11925
11926	case OP_POPF:
11927	{
11928	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11929	uint32_t cbParm;
11930	uint32_t uMask;
11931	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11932	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11933	{
11934	cbParm = 4;
11935	uMask = 0xffffffff;
11936	}
11937	else
11938	{
11939	cbParm = 2;
11940	uMask = 0xffff;
11941	}
11942
11943	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11944	RTGCPTR GCPtrStack = 0;
11945	X86EFLAGS Eflags;
11946	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11947	&GCPtrStack);
11948	if (RT_SUCCESS(rc))
11949	{
11950	Assert(sizeof(Eflags.u32) >= cbParm);
11951	Eflags.u32 = 0;
11952	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
11953	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
11954	}
11955	if (RT_FAILURE(rc))
11956	{
11957	rc = VERR_EM_INTERPRETER;
11958	break;
11959	}
11960	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11961	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
11962	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11963	pMixedCtx->esp += cbParm;
11964	pMixedCtx->esp &= uMask;
11965	pMixedCtx->rip += pDis->cbInstr;
11966	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11967	\| HM_CHANGED_GUEST_RSP
11968	\| HM_CHANGED_GUEST_RFLAGS);
11969	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
11970	POPF restores EFLAGS.TF. */
11971	if ( !fDbgStepping
11972	&& fGstStepping)
11973	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11975	break;
11976	}
11977
11978	case OP_PUSHF:
11979	{
11980	uint32_t cbParm;
11981	uint32_t uMask;
11982	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11983	{
11984	cbParm = 4;
11985	uMask = 0xffffffff;
11986	}
11987	else
11988	{
11989	cbParm = 2;
11990	uMask = 0xffff;
11991	}
11992
11993	/* Get the stack pointer & push the contents of eflags onto the stack. */
11994	RTGCPTR GCPtrStack = 0;
11995	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11996	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11997	if (RT_FAILURE(rc))
11998	{
11999	rc = VERR_EM_INTERPRETER;
12000	break;
12001	}
12002	X86EFLAGS Eflags = pMixedCtx->eflags;
12003	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
12004	Eflags.Bits.u1RF = 0;
12005	Eflags.Bits.u1VM = 0;
12006
12007	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
12008	if (RT_UNLIKELY(rc != VINF_SUCCESS))
12009	{
12010	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
12011	rc = VERR_EM_INTERPRETER;
12012	break;
12013	}
12014	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
12015	pMixedCtx->esp -= cbParm;
12016	pMixedCtx->esp &= uMask;
12017	pMixedCtx->rip += pDis->cbInstr;
12018	pMixedCtx->eflags.Bits.u1RF = 0;
12019	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12020	\| HM_CHANGED_GUEST_RSP
12021	\| HM_CHANGED_GUEST_RFLAGS);
12022	if ( !fDbgStepping
12023	&& pMixedCtx->eflags.Bits.u1TF)
12024	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12025	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
12026	break;
12027	}
12028
12029	case OP_IRET:
12030	{
12031	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
12032	* instruction reference. */
12033	RTGCPTR GCPtrStack = 0;
12034	uint32_t uMask = 0xffff;
12035	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12036	uint16_t aIretFrame[3];
12037	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
12038	{
12039	rc = VERR_EM_INTERPRETER;
12040	break;
12041	}
12042	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12043	&GCPtrStack);
12044	if (RT_SUCCESS(rc))
12045	{
12046	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
12047	PGMACCESSORIGIN_HM));
12048	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
12049	}
12050	if (RT_FAILURE(rc))
12051	{
12052	rc = VERR_EM_INTERPRETER;
12053	break;
12054	}
12055	pMixedCtx->eip = 0;
12056	pMixedCtx->ip = aIretFrame[0];
12057	pMixedCtx->cs.Sel = aIretFrame[1];
12058	pMixedCtx->cs.ValidSel = aIretFrame[1];
12059	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12060	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
12061	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12062	pMixedCtx->sp += sizeof(aIretFrame);
12063	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12064	\| HM_CHANGED_GUEST_SEGMENT_REGS
12065	\| HM_CHANGED_GUEST_RSP
12066	\| HM_CHANGED_GUEST_RFLAGS);
12067	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12068	if ( !fDbgStepping
12069	&& fGstStepping)
12070	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12071	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12072	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12073	break;
12074	}
12075
12076	case OP_INT:
12077	{
12078	uint16_t uVector = pDis->Param1.uValue & 0xff;
12079	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12080	/* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12081	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12082	break;
12083	}
12084
12085	case OP_INTO:
12086	{
12087	if (pMixedCtx->eflags.Bits.u1OF)
12088	{
12089	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12090	/* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12092	}
12093	else
12094	{
12095	pMixedCtx->eflags.Bits.u1RF = 0;
12096	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12097	}
12098	break;
12099	}
12100
12101	default:
12102	{
12103	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12104	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12105	EMCODETYPE_SUPERVISOR);
12106	rc = VBOXSTRICTRC_VAL(rc2);
12107	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12108	/** @todo We have to set pending-debug exceptions here when the guest is
12109	* single-stepping depending on the instruction that was interpreted. */
12110	Log4(("#GP rc=%Rrc\n", rc));
12111	break;
12112	}
12113	}
12114	}
12115	else
12116	rc = VERR_EM_INTERPRETER;
12117
12118	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
12119	("#GP Unexpected rc=%Rrc\n", rc));
12120	return rc;
12121	}
12122
12123
12124	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12125	/**
12126	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12127	* the exception reported in the VMX transient structure back into the VM.
12128	*
12129	* @remarks Requires uExitIntInfo in the VMX transient structure to be
12130	* up-to-date.
12131	*/
12132	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12133	{
12134	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12135
12136	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12137	hmR0VmxCheckExitDueToEventDelivery(). */
12138	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12139	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12140	AssertRCReturn(rc, rc);
12141	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12142
12143	#ifdef DEBUG_ramshankar
12144	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12145	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12146	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12147	#endif
12148
12149	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12150	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12151	return VINF_SUCCESS;
12152	}
12153	#endif
12154
12155
12156	/**
12157	* VM-exit exception handler for \#PF (Page-fault exception).
12158	*/
12159	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12160	{
12161	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12162	PVM pVM = pVCpu->CTX_SUFF(pVM);
12163	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12164	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12165	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12166	AssertRCReturn(rc, rc);
12167
12168	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
12169	if (pVM->hm.s.fNestedPaging)
12170	{
12171	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12172	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12173	{
12174	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12175	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12176	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12177	}
12178	else
12179	{
12180	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12181	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12182	Log4(("Pending #DF due to vectoring #PF. NP\n"));
12183	}
12184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12185	return rc;
12186	}
12187	#else
12188	Assert(!pVM->hm.s.fNestedPaging);
12189	NOREF(pVM);
12190	#endif
12191
12192	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12193	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12194	if (pVmxTransient->fVectoringPF)
12195	{
12196	Assert(pVCpu->hm.s.Event.fPending);
12197	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12198	}
12199
12200	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12201	AssertRCReturn(rc, rc);
12202
12203	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12204	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12205
12206	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12207	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12208	(RTGCPTR)pVmxTransient->uExitQualification);
12209
12210	Log4(("#PF: rc=%Rrc\n", rc));
12211	if (rc == VINF_SUCCESS)
12212	{
12213	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12214	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
12215	* memory? We don't update the whole state here... */
12216	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12217	\| HM_CHANGED_GUEST_RSP
12218	\| HM_CHANGED_GUEST_RFLAGS
12219	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
12220	TRPMResetTrap(pVCpu);
12221	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12222	return rc;
12223	}
12224
12225	if (rc == VINF_EM_RAW_GUEST_TRAP)
12226	{
12227	if (!pVmxTransient->fVectoringDoublePF)
12228	{
12229	/* It's a guest page fault and needs to be reflected to the guest. */
12230	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12231	TRPMResetTrap(pVCpu);
12232	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12233	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12234	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12235	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12236	}
12237	else
12238	{
12239	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12240	TRPMResetTrap(pVCpu);
12241	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12242	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12243	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12244	}
12245
12246	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12247	return VINF_SUCCESS;
12248	}
12249
12250	TRPMResetTrap(pVCpu);
12251	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12252	return rc;
12253	}
12254
12255	/** @} */
12256

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 58944

Download in other formats: