HMVMXR0.cpp@ 50614

Last change on this file since 50614 was 50614, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Avoid one extra VM-exit while injecting interrupts if interrupt-window exiting was previously there but exited for other reasons.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 476.7 KB

Line
1	/* $Id: HMVMXR0.cpp 50614 2014-02-26 18:19:18Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns true if the MSR was added -and- its value was updated, false
1239	* otherwise.
1240	* @param pVCpu Pointer to the VMCPU.
1241	* @param uMsr The MSR.
1242	* @param uGuestMsr Value of the guest MSR.
1243	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1244	* necessary.
1245	*/
1246	static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1247	{
1248	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1249	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1250	uint32_t i;
1251	for (i = 0; i < cMsrs; i++)
1252	{
1253	if (pGuestMsr->u32Msr == uMsr)
1254	break;
1255	pGuestMsr++;
1256	}
1257
1258	bool fAdded = false;
1259	if (i == cMsrs)
1260	{
1261	++cMsrs;
1262	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1263	AssertRC(rc);
1264
1265	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1266	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1267	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1268
1269	fAdded = true;
1270	}
1271
1272	/* Update the MSR values in the auto-load/store MSR area. */
1273	pGuestMsr->u32Msr = uMsr;
1274	pGuestMsr->u64Value = uGuestMsrValue;
1275
1276	/* Create/update the MSR slot in the host MSR area. */
1277	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1278	pHostMsr += i;
1279	pHostMsr->u32Msr = uMsr;
1280
1281	/*
1282	* Update the host MSR only when requested by the caller AND when we're
1283	* adding it to the auto-load/store area. Otherwise, it would have been
1284	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1285	*/
1286	bool fUpdatedMsrValue = false;
1287	if ( fAdded
1288	&& fUpdateHostMsr)
1289	{
1290	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1291	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1292	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1293	fUpdatedMsrValue = true;
1294	}
1295
1296	return fUpdatedMsrValue;
1297	}
1298
1299
1300	/**
1301	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1302	* auto-load/store MSR area in the VMCS.
1303	*
1304	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1305	* area.
1306	*
1307	* @returns VBox status code.
1308	* @param pVCpu Pointer to the VMCPU.
1309	* @param uMsr The MSR.
1310	*/
1311	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1312	{
1313	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1314	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1315	for (uint32_t i = 0; i < cMsrs; i++)
1316	{
1317	/* Find the MSR. */
1318	if (pGuestMsr->u32Msr == uMsr)
1319	{
1320	/* If it's the last MSR, simply reduce the count. */
1321	if (i == cMsrs - 1)
1322	{
1323	--cMsrs;
1324	break;
1325	}
1326
1327	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1328	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1329	pLastGuestMsr += cMsrs;
1330	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1331	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1332
1333	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1334	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1335	pLastHostMsr += cMsrs;
1336	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1337	pHostMsr->u64Value = pLastHostMsr->u64Value;
1338	--cMsrs;
1339	break;
1340	}
1341	pGuestMsr++;
1342	}
1343
1344	/* Update the VMCS if the count changed (meaning the MSR was found). */
1345	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1346	{
1347	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1348	AssertRCReturn(rc, rc);
1349
1350	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1351	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1352	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1353	}
1354
1355	return VINF_SUCCESS;
1356	}
1357
1358
1359	/**
1360	* Checks if the specified guest MSR is part of the auto-load/store area in
1361	* the VMCS.
1362	*
1363	* @returns true if found, false otherwise.
1364	* @param pVCpu Pointer to the VMCPU.
1365	* @param uMsr The MSR to find.
1366	*/
1367	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1368	{
1369	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1370	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1371
1372	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1373	{
1374	if (pGuestMsr->u32Msr == uMsr)
1375	return true;
1376	}
1377	return false;
1378	}
1379
1380
1381	/**
1382	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1383	*
1384	* @param pVCpu Pointer to the VMCPU.
1385	*
1386	* @remarks No-long-jump zone!!!
1387	*/
1388	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1389	{
1390	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1391	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1392	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1393	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1394
1395	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1396	{
1397	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1398	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1399	}
1400
1401	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1402	}
1403
1404
1405	#if HC_ARCH_BITS == 64
1406	/**
1407	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1408	* perform lazy restoration of the host MSRs while leaving VT-x.
1409	*
1410	* @param pVCpu Pointer to the VMCPU.
1411	*
1412	* @remarks No-long-jump zone!!!
1413	*/
1414	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1415	{
1416	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1417
1418	/*
1419	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1420	*/
1421	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1422	{
1423	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1424	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1425	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1426	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1427	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1428	}
1429	}
1430
1431
1432	/**
1433	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1434	* lazily while leaving VT-x.
1435	*
1436	* @returns true if it does, false otherwise.
1437	* @param pVCpu Pointer to the VMCPU.
1438	* @param uMsr The MSR to check.
1439	*/
1440	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1441	{
1442	NOREF(pVCpu);
1443	switch (uMsr)
1444	{
1445	case MSR_K8_LSTAR:
1446	case MSR_K6_STAR:
1447	case MSR_K8_SF_MASK:
1448	case MSR_K8_KERNEL_GS_BASE:
1449	return true;
1450	}
1451	return false;
1452	}
1453
1454
1455	/**
1456	* Saves a set of guests MSRs back into the guest-CPU context.
1457	*
1458	* @param pVCpu Pointer to the VMCPU.
1459	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1460	* out-of-sync. Make sure to update the required fields
1461	* before using them.
1462	*
1463	* @remarks No-long-jump zone!!!
1464	*/
1465	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1466	{
1467	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1468	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1469
1470	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1471	{
1472	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1473	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1474	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1475	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1476	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1477	}
1478	}
1479
1480
1481	/**
1482	* Loads a set of guests MSRs to allow read/passthru to the guest.
1483	*
1484	* The name of this function is slightly confusing. This function does NOT
1485	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1486	* common prefix for functions dealing with "lazy restoration" of the shared
1487	* MSRs.
1488	*
1489	* @param pVCpu Pointer to the VMCPU.
1490	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1491	* out-of-sync. Make sure to update the required fields
1492	* before using them.
1493	*
1494	* @remarks No-long-jump zone!!!
1495	*/
1496	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1497	{
1498	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1499	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1500
1501	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1502	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1503	{
1504	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1505	do { \
1506	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1507	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1508	else \
1509	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1510	} while (0)
1511
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1513	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1514	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1515	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1516	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1517	}
1518	else
1519	{
1520	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1521	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1522	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1523	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1524	}
1525	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1526	}
1527
1528
1529	/**
1530	* Performs lazy restoration of the set of host MSRs if they were previously
1531	* loaded with guest MSR values.
1532	*
1533	* @param pVCpu Pointer to the VMCPU.
1534	*
1535	* @remarks No-long-jump zone!!!
1536	* @remarks The guest MSRs should have been saved back into the guest-CPU
1537	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1538	*/
1539	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1540	{
1541	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1542	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1543
1544	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1545	{
1546	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1547	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1548	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1549	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1550	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1551	}
1552	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1553	}
1554	#endif /* HC_ARCH_BITS == 64 */
1555
1556
1557	#ifdef VBOX_STRICT
1558	/**
1559	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1560	* VMCS are correct.
1561	*
1562	* @param pVCpu Pointer to the VMCPU.
1563	*/
1564	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1565	{
1566	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1567
1568	/* Verify MSR counts in the VMCS are what we think it should be. */
1569	uint32_t cMsrs;
1570	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1577	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1578
1579	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1580	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1581	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1582	{
1583	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1584	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1585	pGuestMsr->u32Msr));
1586
1587	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1588	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1589	pHostMsr->u64Value, u64Msr));
1590
1591	/* Verify that the permissions are as expected in the MSR bitmap. */
1592	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1593	{
1594	VMXMSREXITREAD enmRead;
1595	VMXMSREXITWRITE enmWrite;
1596	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1597	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1598	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1599	pGuestMsr->u32Msr));
1600	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1601	pGuestMsr->u32Msr));
1602	}
1603	}
1604	}
1605	# endif /* VBOX_STRICT */
1606
1607
1608	/**
1609	* Flushes the TLB using EPT.
1610	*
1611	* @returns VBox status code.
1612	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1613	* enmFlush).
1614	* @param enmFlush Type of flush.
1615	*
1616	* @remarks Caller is responsible for making sure this function is called only
1617	* when NestedPaging is supported and providing @a enmFlush that is
1618	* supported by the CPU.
1619	* @remarks Can be called with interrupts disabled.
1620	*/
1621	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1622	{
1623	uint64_t au64Descriptor[2];
1624	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1625	au64Descriptor[0] = 0;
1626	else
1627	{
1628	Assert(pVCpu);
1629	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1630	}
1631	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1632
1633	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1634	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1635	rc));
1636	if ( RT_SUCCESS(rc)
1637	&& pVCpu)
1638	{
1639	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1640	}
1641	}
1642
1643
1644	/**
1645	* Flushes the TLB using VPID.
1646	*
1647	* @returns VBox status code.
1648	* @param pVM Pointer to the VM.
1649	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1650	* enmFlush).
1651	* @param enmFlush Type of flush.
1652	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1653	* on @a enmFlush).
1654	*
1655	* @remarks Can be called with interrupts disabled.
1656	*/
1657	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1658	{
1659	NOREF(pVM);
1660	AssertPtr(pVM);
1661	Assert(pVM->hm.s.vmx.fVpid);
1662
1663	uint64_t au64Descriptor[2];
1664	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1665	{
1666	au64Descriptor[0] = 0;
1667	au64Descriptor[1] = 0;
1668	}
1669	else
1670	{
1671	AssertPtr(pVCpu);
1672	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1673	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1674	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1675	au64Descriptor[1] = GCPtr;
1676	}
1677
1678	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1679	AssertMsg(rc == VINF_SUCCESS,
1680	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1681	if ( RT_SUCCESS(rc)
1682	&& pVCpu)
1683	{
1684	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1685	}
1686	}
1687
1688
1689	/**
1690	* Invalidates a guest page by guest virtual address. Only relevant for
1691	* EPT/VPID, otherwise there is nothing really to invalidate.
1692	*
1693	* @returns VBox status code.
1694	* @param pVM Pointer to the VM.
1695	* @param pVCpu Pointer to the VMCPU.
1696	* @param GCVirt Guest virtual address of the page to invalidate.
1697	*/
1698	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1699	{
1700	AssertPtr(pVM);
1701	AssertPtr(pVCpu);
1702	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1703
1704	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1705	if (!fFlushPending)
1706	{
1707	/*
1708	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1709	* See @bugref{6043} and @bugref{6177}.
1710	*
1711	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1712	* function maybe called in a loop with individual addresses.
1713	*/
1714	if (pVM->hm.s.vmx.fVpid)
1715	{
1716	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1717	{
1718	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1719	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1720	}
1721	else
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724	else if (pVM->hm.s.fNestedPaging)
1725	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1726	}
1727
1728	return VINF_SUCCESS;
1729	}
1730
1731
1732	/**
1733	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1734	* otherwise there is nothing really to invalidate.
1735	*
1736	* @returns VBox status code.
1737	* @param pVM Pointer to the VM.
1738	* @param pVCpu Pointer to the VMCPU.
1739	* @param GCPhys Guest physical address of the page to invalidate.
1740	*/
1741	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1742	{
1743	NOREF(pVM); NOREF(GCPhys);
1744	LogFlowFunc(("%RGp\n", GCPhys));
1745
1746	/*
1747	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1748	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1749	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1750	*/
1751	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1752	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1753	return VINF_SUCCESS;
1754	}
1755
1756
1757	/**
1758	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1759	* case where neither EPT nor VPID is supported by the CPU.
1760	*
1761	* @param pVM Pointer to the VM.
1762	* @param pVCpu Pointer to the VMCPU.
1763	* @param pCpu Pointer to the global HM struct.
1764	*
1765	* @remarks Called with interrupts disabled.
1766	*/
1767	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1768	{
1769	AssertPtr(pVCpu);
1770	AssertPtr(pCpu);
1771	NOREF(pVM);
1772
1773	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1774
1775	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1776	#if 0
1777	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1778	pVCpu->hm.s.TlbShootdown.cPages = 0;
1779	#endif
1780
1781	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1782	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1783	pVCpu->hm.s.fForceTLBFlush = false;
1784	return;
1785	}
1786
1787
1788	/**
1789	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1790	*
1791	* @param pVM Pointer to the VM.
1792	* @param pVCpu Pointer to the VMCPU.
1793	* @param pCpu Pointer to the global HM CPU struct.
1794	* @remarks All references to "ASID" in this function pertains to "VPID" in
1795	* Intel's nomenclature. The reason is, to avoid confusion in compare
1796	* statements since the host-CPU copies are named "ASID".
1797	*
1798	* @remarks Called with interrupts disabled.
1799	*/
1800	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1801	{
1802	#ifdef VBOX_WITH_STATISTICS
1803	bool fTlbFlushed = false;
1804	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1805	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1806	if (!fTlbFlushed) \
1807	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1808	} while (0)
1809	#else
1810	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1811	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1812	#endif
1813
1814	AssertPtr(pVM);
1815	AssertPtr(pCpu);
1816	AssertPtr(pVCpu);
1817	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1818	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1819	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1820
1821
1822	/*
1823	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1824	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1825	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1826	*/
1827	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1828	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1829	{
1830	++pCpu->uCurrentAsid;
1831	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1832	{
1833	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1834	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1835	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1836	}
1837
1838	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1839	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1840	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1841
1842	/*
1843	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1844	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1845	*/
1846	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1847	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1848	HMVMX_SET_TAGGED_TLB_FLUSHED();
1849	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1850	}
1851
1852	/* Check for explicit TLB shootdowns. */
1853	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1854	{
1855	/*
1856	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1857	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1858	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1859	* but not guest-physical mappings.
1860	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1861	*/
1862	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1863	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1864	HMVMX_SET_TAGGED_TLB_FLUSHED();
1865	}
1866
1867	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1868	* where it is commented out. Support individual entry flushing
1869	* someday. */
1870	#if 0
1871	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1872	{
1873	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1874
1875	/*
1876	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1877	* as supported by the CPU.
1878	*/
1879	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1880	{
1881	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1882	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1883	}
1884	else
1885	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1886
1887	HMVMX_SET_TAGGED_TLB_FLUSHED();
1888	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1889	pVCpu->hm.s.TlbShootdown.cPages = 0;
1890	}
1891	#endif
1892
1893	pVCpu->hm.s.fForceTLBFlush = false;
1894
1895	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1896
1897	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1898	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1899	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1900	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1901	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1902	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1903	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1904	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1905
1906	/* Update VMCS with the VPID. */
1907	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1908	AssertRC(rc);
1909
1910	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1911	}
1912
1913
1914	/**
1915	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1916	*
1917	* @returns VBox status code.
1918	* @param pVM Pointer to the VM.
1919	* @param pVCpu Pointer to the VMCPU.
1920	* @param pCpu Pointer to the global HM CPU struct.
1921	*
1922	* @remarks Called with interrupts disabled.
1923	*/
1924	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1925	{
1926	AssertPtr(pVM);
1927	AssertPtr(pVCpu);
1928	AssertPtr(pCpu);
1929	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1930	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1931
1932	/*
1933	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1934	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1935	*/
1936	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1937	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1938	{
1939	pVCpu->hm.s.fForceTLBFlush = true;
1940	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1941	}
1942
1943	/* Check for explicit TLB shootdown flushes. */
1944	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1945	{
1946	pVCpu->hm.s.fForceTLBFlush = true;
1947	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1948	}
1949
1950	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1951	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1952
1953	if (pVCpu->hm.s.fForceTLBFlush)
1954	{
1955	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1956	pVCpu->hm.s.fForceTLBFlush = false;
1957	}
1958	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1959	* where it is commented out. Support individual entry flushing
1960	* someday. */
1961	#if 0
1962	else
1963	{
1964	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1965	{
1966	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1967	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1968	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1969	}
1970	else
1971	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1972
1973	pVCpu->hm.s.TlbShootdown.cPages = 0;
1974	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1975	}
1976	#endif
1977	}
1978
1979
1980	/**
1981	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1982	*
1983	* @returns VBox status code.
1984	* @param pVM Pointer to the VM.
1985	* @param pVCpu Pointer to the VMCPU.
1986	* @param pCpu Pointer to the global HM CPU struct.
1987	*
1988	* @remarks Called with interrupts disabled.
1989	*/
1990	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1991	{
1992	AssertPtr(pVM);
1993	AssertPtr(pVCpu);
1994	AssertPtr(pCpu);
1995	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1996	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1997
1998	/*
1999	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2000	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2001	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2002	*/
2003	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2004	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2005	{
2006	pVCpu->hm.s.fForceTLBFlush = true;
2007	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2008	}
2009
2010	/* Check for explicit TLB shootdown flushes. */
2011	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2012	{
2013	/*
2014	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2015	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2016	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2017	*/
2018	pVCpu->hm.s.fForceTLBFlush = true;
2019	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2020	}
2021
2022	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2023	if (pVCpu->hm.s.fForceTLBFlush)
2024	{
2025	++pCpu->uCurrentAsid;
2026	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2027	{
2028	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2029	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2030	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2031	}
2032
2033	pVCpu->hm.s.fForceTLBFlush = false;
2034	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2035	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2036	if (pCpu->fFlushAsidBeforeUse)
2037	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2038	}
2039	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2040	* where it is commented out. Support individual entry flushing
2041	* someday. */
2042	#if 0
2043	else
2044	{
2045	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2046	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2047	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2048	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2049
2050	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2051	{
2052	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2053	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2054	{
2055	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2056	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2057	}
2058	else
2059	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2060
2061	pVCpu->hm.s.TlbShootdown.cPages = 0;
2062	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2063	}
2064	else
2065	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2066	}
2067	#endif
2068
2069	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2070	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2071	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2072	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2073	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2074	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2075
2076	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2077	AssertRC(rc);
2078	}
2079
2080
2081	/**
2082	* Flushes the guest TLB entry based on CPU capabilities.
2083	*
2084	* @param pVCpu Pointer to the VMCPU.
2085	* @param pCpu Pointer to the global HM CPU struct.
2086	*/
2087	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2088	{
2089	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2090	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2091	#endif
2092	PVM pVM = pVCpu->CTX_SUFF(pVM);
2093	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2094	{
2095	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2096	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2097	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2098	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2099	default:
2100	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2101	break;
2102	}
2103
2104	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2105	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2106
2107	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2108	}
2109
2110
2111	/**
2112	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2113	* TLB entries from the host TLB before VM-entry.
2114	*
2115	* @returns VBox status code.
2116	* @param pVM Pointer to the VM.
2117	*/
2118	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2119	{
2120	/*
2121	* Determine optimal flush type for Nested Paging.
2122	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2123	* guest execution (see hmR3InitFinalizeR0()).
2124	*/
2125	if (pVM->hm.s.fNestedPaging)
2126	{
2127	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2128	{
2129	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2130	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2131	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2132	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2133	else
2134	{
2135	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2136	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2137	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2138	}
2139
2140	/* Make sure the write-back cacheable memory type for EPT is supported. */
2141	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2142	{
2143	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2144	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2145	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2146	}
2147	}
2148	else
2149	{
2150	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2151	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2152	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2153	}
2154	}
2155
2156	/*
2157	* Determine optimal flush type for VPID.
2158	*/
2159	if (pVM->hm.s.vmx.fVpid)
2160	{
2161	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2162	{
2163	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2164	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2165	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2166	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2167	else
2168	{
2169	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2170	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2171	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2172	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2173	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2174	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2175	pVM->hm.s.vmx.fVpid = false;
2176	}
2177	}
2178	else
2179	{
2180	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2181	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2182	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2183	pVM->hm.s.vmx.fVpid = false;
2184	}
2185	}
2186
2187	/*
2188	* Setup the handler for flushing tagged-TLBs.
2189	*/
2190	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2191	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2192	else if (pVM->hm.s.fNestedPaging)
2193	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2194	else if (pVM->hm.s.vmx.fVpid)
2195	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2196	else
2197	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2198	return VINF_SUCCESS;
2199	}
2200
2201
2202	/**
2203	* Sets up pin-based VM-execution controls in the VMCS.
2204	*
2205	* @returns VBox status code.
2206	* @param pVM Pointer to the VM.
2207	* @param pVCpu Pointer to the VMCPU.
2208	*/
2209	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2210	{
2211	AssertPtr(pVM);
2212	AssertPtr(pVCpu);
2213
2214	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2215	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2216
2217	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2218	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2219	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2220
2221	/* Enable the VMX preemption timer. */
2222	if (pVM->hm.s.vmx.fUsePreemptTimer)
2223	{
2224	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2225	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2226	}
2227
2228	if ((val & zap) != val)
2229	{
2230	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2231	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2232	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2233	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234	}
2235
2236	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2237	AssertRCReturn(rc, rc);
2238
2239	/* Update VCPU with the currently set pin-based VM-execution controls. */
2240	pVCpu->hm.s.vmx.u32PinCtls = val;
2241	return rc;
2242	}
2243
2244
2245	/**
2246	* Sets up processor-based VM-execution controls in the VMCS.
2247	*
2248	* @returns VBox status code.
2249	* @param pVM Pointer to the VM.
2250	* @param pVMCPU Pointer to the VMCPU.
2251	*/
2252	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2253	{
2254	AssertPtr(pVM);
2255	AssertPtr(pVCpu);
2256
2257	int rc = VERR_INTERNAL_ERROR_5;
2258	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2259	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2260
2261	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2265	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2266	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2267	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2268
2269	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2270	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2271	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2272	{
2273	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2274	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2275	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2276	}
2277
2278	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2279	if (!pVM->hm.s.fNestedPaging)
2280	{
2281	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2282	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2283	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2284	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2285	}
2286
2287	/* Use TPR shadowing if supported by the CPU. */
2288	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2289	{
2290	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2292	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2293	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2294	AssertRCReturn(rc, rc);
2295
2296	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2297	/* CR8 writes causes a VM-exit based on TPR threshold. */
2298	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2299	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2300	}
2301	else
2302	{
2303	/*
2304	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2305	* Set this control only for 64-bit guests.
2306	*/
2307	if (pVM->hm.s.fAllow64BitGuests)
2308	{
2309	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2310	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2311	}
2312	}
2313
2314	/* Use MSR-bitmaps if supported by the CPU. */
2315	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2316	{
2317	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2318
2319	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2320	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2321	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2322	AssertRCReturn(rc, rc);
2323
2324	/*
2325	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2326	* automatically as dedicated fields in the VMCS.
2327	*/
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2331	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2332	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2333
2334	#if HC_ARCH_BITS == 64
2335	/*
2336	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2337	*/
2338	if (pVM->hm.s.fAllow64BitGuests)
2339	{
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2342	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2343	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2344	}
2345	#endif
2346	}
2347
2348	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2349	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2350	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2351
2352	if ((val & zap) != val)
2353	{
2354	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2355	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2356	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2357	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2358	}
2359
2360	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2361	AssertRCReturn(rc, rc);
2362
2363	/* Update VCPU with the currently set processor-based VM-execution controls. */
2364	pVCpu->hm.s.vmx.u32ProcCtls = val;
2365
2366	/*
2367	* Secondary processor-based VM-execution controls.
2368	*/
2369	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2370	{
2371	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2372	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2373
2374	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2376
2377	if (pVM->hm.s.fNestedPaging)
2378	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2379	else
2380	{
2381	/*
2382	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2383	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2384	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2385	*/
2386	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2387	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2388	}
2389
2390	if (pVM->hm.s.vmx.fVpid)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2392
2393	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2394	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2395
2396	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2397	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2398	* done dynamically. */
2399	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2400	{
2401	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2402	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2403	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2404	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2405	AssertRCReturn(rc, rc);
2406	}
2407
2408	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2409	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2410
2411	if ((val & zap) != val)
2412	{
2413	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2414	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2415	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2416	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2417	}
2418
2419	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2420	AssertRCReturn(rc, rc);
2421
2422	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2423	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2424	}
2425	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2426	{
2427	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2428	"available\n"));
2429	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2430	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2431	}
2432
2433	return VINF_SUCCESS;
2434	}
2435
2436
2437	/**
2438	* Sets up miscellaneous (everything other than Pin & Processor-based
2439	* VM-execution) control fields in the VMCS.
2440	*
2441	* @returns VBox status code.
2442	* @param pVM Pointer to the VM.
2443	* @param pVCpu Pointer to the VMCPU.
2444	*/
2445	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2446	{
2447	NOREF(pVM);
2448	AssertPtr(pVM);
2449	AssertPtr(pVCpu);
2450
2451	int rc = VERR_GENERAL_FAILURE;
2452
2453	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2454	#if 0
2455	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2456	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2457	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2458
2459	/*
2460	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2461	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2462	* We thus use the exception bitmap to control it rather than use both.
2463	*/
2464	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2465	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2466
2467	/** @todo Explore possibility of using IO-bitmaps. */
2468	/* All IO & IOIO instructions cause VM-exits. */
2469	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2470	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2471
2472	/* Initialize the MSR-bitmap area. */
2473	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2474	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2476	#endif
2477
2478	/* Setup MSR auto-load/store area. */
2479	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2480	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2481	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2482	AssertRCReturn(rc, rc);
2483	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2484	AssertRCReturn(rc, rc);
2485
2486	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2487	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2488	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2489	AssertRCReturn(rc, rc);
2490
2491	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2492	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2493	AssertRCReturn(rc, rc);
2494
2495	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2496	#if 0
2497	/* Setup debug controls */
2498	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2499	AssertRCReturn(rc, rc);
2500	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2501	AssertRCReturn(rc, rc);
2502	#endif
2503
2504	return rc;
2505	}
2506
2507
2508	/**
2509	* Sets up the initial exception bitmap in the VMCS based on static conditions
2510	* (i.e. conditions that cannot ever change after starting the VM).
2511	*
2512	* @returns VBox status code.
2513	* @param pVM Pointer to the VM.
2514	* @param pVCpu Pointer to the VMCPU.
2515	*/
2516	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2517	{
2518	AssertPtr(pVM);
2519	AssertPtr(pVCpu);
2520
2521	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2522
2523	uint32_t u32XcptBitmap = 0;
2524
2525	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2526	if (!pVM->hm.s.fNestedPaging)
2527	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2528
2529	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2530	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2531	AssertRCReturn(rc, rc);
2532	return rc;
2533	}
2534
2535
2536	/**
2537	* Sets up the initial guest-state mask. The guest-state mask is consulted
2538	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2539	* for the nested virtualization case (as it would cause a VM-exit).
2540	*
2541	* @param pVCpu Pointer to the VMCPU.
2542	*/
2543	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2544	{
2545	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2546	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2547	return VINF_SUCCESS;
2548	}
2549
2550
2551	/**
2552	* Does per-VM VT-x initialization.
2553	*
2554	* @returns VBox status code.
2555	* @param pVM Pointer to the VM.
2556	*/
2557	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2558	{
2559	LogFlowFunc(("pVM=%p\n", pVM));
2560
2561	int rc = hmR0VmxStructsAlloc(pVM);
2562	if (RT_FAILURE(rc))
2563	{
2564	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2565	return rc;
2566	}
2567
2568	return VINF_SUCCESS;
2569	}
2570
2571
2572	/**
2573	* Does per-VM VT-x termination.
2574	*
2575	* @returns VBox status code.
2576	* @param pVM Pointer to the VM.
2577	*/
2578	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2579	{
2580	LogFlowFunc(("pVM=%p\n", pVM));
2581
2582	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2583	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2584	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2585	#endif
2586	hmR0VmxStructsFree(pVM);
2587	return VINF_SUCCESS;
2588	}
2589
2590
2591	/**
2592	* Sets up the VM for execution under VT-x.
2593	* This function is only called once per-VM during initialization.
2594	*
2595	* @returns VBox status code.
2596	* @param pVM Pointer to the VM.
2597	*/
2598	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2599	{
2600	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2601	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2602
2603	LogFlowFunc(("pVM=%p\n", pVM));
2604
2605	/*
2606	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2607	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2608	*/
2609	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2610	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2611	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2612	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2613	{
2614	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2615	return VERR_INTERNAL_ERROR;
2616	}
2617
2618	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2619	/*
2620	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2621	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2622	*/
2623	if ( pVM->hm.s.fAllow64BitGuests
2624	&& !HMVMX_IS_64BIT_HOST_MODE())
2625	{
2626	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2627	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2628	}
2629	#endif
2630
2631	/* Initialize these always, see hmR3InitFinalizeR0().*/
2632	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2633	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2634
2635	/* Setup the tagged-TLB flush handlers. */
2636	int rc = hmR0VmxSetupTaggedTlb(pVM);
2637	if (RT_FAILURE(rc))
2638	{
2639	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2640	return rc;
2641	}
2642
2643	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2644	{
2645	PVMCPU pVCpu = &pVM->aCpus[i];
2646	AssertPtr(pVCpu);
2647	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2648
2649	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2650	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2651
2652	/* Set revision dword at the beginning of the VMCS structure. */
2653	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2654
2655	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2656	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2657	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2658	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2659
2660	/* Load this VMCS as the current VMCS. */
2661	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2662	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2663	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2664
2665	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2666	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2667	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2668
2669	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2670	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2671	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2672
2673	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2674	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2675	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2676
2677	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2678	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2679	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2680
2681	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2682	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2683	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2684
2685	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2686	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2687	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2688	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2689	#endif
2690
2691	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2692	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2693	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2694	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2695
2696	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2697
2698	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2699	}
2700
2701	return VINF_SUCCESS;
2702	}
2703
2704
2705	/**
2706	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2707	* the VMCS.
2708	*
2709	* @returns VBox status code.
2710	* @param pVM Pointer to the VM.
2711	* @param pVCpu Pointer to the VMCPU.
2712	*/
2713	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2714	{
2715	NOREF(pVM); NOREF(pVCpu);
2716
2717	RTCCUINTREG uReg = ASMGetCR0();
2718	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2719	AssertRCReturn(rc, rc);
2720
2721	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2722	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2723	if (HMVMX_IS_64BIT_HOST_MODE())
2724	{
2725	uint64_t uRegCR3 = HMR0Get64bitCR3();
2726	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2727	}
2728	else
2729	#endif
2730	{
2731	uReg = ASMGetCR3();
2732	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2733	}
2734	AssertRCReturn(rc, rc);
2735
2736	uReg = ASMGetCR4();
2737	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2738	AssertRCReturn(rc, rc);
2739	return rc;
2740	}
2741
2742
2743	#if HC_ARCH_BITS == 64
2744	/**
2745	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2746	* requirements. See hmR0VmxSaveHostSegmentRegs().
2747	*/
2748	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2749	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2750	{ \
2751	bool fValidSelector = true; \
2752	if ((selValue) & X86_SEL_LDT) \
2753	{ \
2754	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2755	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2756	} \
2757	if (fValidSelector) \
2758	{ \
2759	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2760	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2761	} \
2762	(selValue) = 0; \
2763	}
2764	#endif
2765
2766
2767	/**
2768	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2769	* the host-state area in the VMCS.
2770	*
2771	* @returns VBox status code.
2772	* @param pVM Pointer to the VM.
2773	* @param pVCpu Pointer to the VMCPU.
2774	*/
2775	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2776	{
2777	NOREF(pVM);
2778	int rc = VERR_INTERNAL_ERROR_5;
2779
2780	#if HC_ARCH_BITS == 64
2781	/*
2782	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2783	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2784	*/
2785	AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2786	("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2787	#endif
2788
2789	/*
2790	* Host DS, ES, FS and GS segment registers.
2791	*/
2792	#if HC_ARCH_BITS == 64
2793	RTSEL uSelDS = ASMGetDS();
2794	RTSEL uSelES = ASMGetES();
2795	RTSEL uSelFS = ASMGetFS();
2796	RTSEL uSelGS = ASMGetGS();
2797	#else
2798	RTSEL uSelDS = 0;
2799	RTSEL uSelES = 0;
2800	RTSEL uSelFS = 0;
2801	RTSEL uSelGS = 0;
2802	#endif
2803
2804	/* Recalculate which host-state bits need to be manually restored. */
2805	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2806
2807	/*
2808	* Host CS and SS segment registers.
2809	*/
2810	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2811	RTSEL uSelCS;
2812	RTSEL uSelSS;
2813	if (HMVMX_IS_64BIT_HOST_MODE())
2814	{
2815	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2816	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2817	}
2818	else
2819	{
2820	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2821	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2822	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2823	}
2824	#else
2825	RTSEL uSelCS = ASMGetCS();
2826	RTSEL uSelSS = ASMGetSS();
2827	#endif
2828
2829	/*
2830	* Host TR segment register.
2831	*/
2832	RTSEL uSelTR = ASMGetTR();
2833
2834	#if HC_ARCH_BITS == 64
2835	/*
2836	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2837	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2838	*/
2839	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2840	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2841	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2842	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2843	# undef VMXLOCAL_ADJUST_HOST_SEG
2844	#endif
2845
2846	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2847	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2848	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2849	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2850	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2851	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2852	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2853	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2854	Assert(uSelCS);
2855	Assert(uSelTR);
2856
2857	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2858	#if 0
2859	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2860	Assert(uSelSS != 0);
2861	#endif
2862
2863	/* Write these host selector fields into the host-state area in the VMCS. */
2864	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2865	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2866	#if HC_ARCH_BITS == 64
2867	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2868	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2869	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2870	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2871	#endif
2872	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2873
2874	/*
2875	* Host GDTR and IDTR.
2876	*/
2877	RTGDTR Gdtr;
2878	RT_ZERO(Gdtr);
2879	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2880	if (HMVMX_IS_64BIT_HOST_MODE())
2881	{
2882	X86XDTR64 Gdtr64;
2883	X86XDTR64 Idtr64;
2884	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2885	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2886	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2887
2888	Gdtr.cbGdt = Gdtr64.cb;
2889	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2890	}
2891	else
2892	#endif
2893	{
2894	RTIDTR Idtr;
2895	ASMGetGDTR(&Gdtr);
2896	ASMGetIDTR(&Idtr);
2897	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2898	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2899
2900	#if HC_ARCH_BITS == 64
2901	/*
2902	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2903	* maximum limit (0xffff) on every VM-exit.
2904	*/
2905	if (Gdtr.cbGdt != 0xffff)
2906	{
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2908	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2909	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2910	}
2911
2912	/*
2913	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2914	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2915	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2916	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2917	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2918	* hosts where we are pretty sure it won't cause trouble.
2919	*/
2920	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
2921	if (Idtr.cbIdt < 0x0fff)
2922	# else
2923	if (Idtr.cbIdt != 0xffff)
2924	# endif
2925	{
2926	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2927	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2928	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2929	}
2930	#endif
2931	}
2932
2933	/*
2934	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2935	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2936	*/
2937	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2938	{
2939	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2940	return VERR_VMX_INVALID_HOST_STATE;
2941	}
2942
2943	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2944	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2945	if (HMVMX_IS_64BIT_HOST_MODE())
2946	{
2947	/* We need the 64-bit TR base for hybrid darwin. */
2948	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2949	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2950	}
2951	else
2952	#endif
2953	{
2954	uintptr_t uTRBase;
2955	#if HC_ARCH_BITS == 64
2956	uTRBase = X86DESC64_BASE(pDesc);
2957
2958	/*
2959	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2960	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2961	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2962	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2963	*
2964	* [1] See Intel spec. 3.5 "System Descriptor Types".
2965	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2966	*/
2967	Assert(pDesc->System.u4Type == 11);
2968	if ( pDesc->System.u16LimitLow != 0x67
2969	\|\| pDesc->System.u4LimitHigh)
2970	{
2971	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2972	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2973
2974	/* Store the GDTR here as we need it while restoring TR. */
2975	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2976	}
2977	#else
2978	uTRBase = X86DESC_BASE(pDesc);
2979	#endif
2980	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2981	}
2982	AssertRCReturn(rc, rc);
2983
2984	/*
2985	* Host FS base and GS base.
2986	*/
2987	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2988	if (HMVMX_IS_64BIT_HOST_MODE())
2989	{
2990	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2991	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2992	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2993	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2994
2995	# if HC_ARCH_BITS == 64
2996	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2997	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2998	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2999	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3000	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3001	# endif
3002	}
3003	#endif
3004	return rc;
3005	}
3006
3007
3008	/**
3009	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3010	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3011	* the host after every successful VM-exit.
3012	*
3013	* @returns VBox status code.
3014	* @param pVM Pointer to the VM.
3015	* @param pVCpu Pointer to the VMCPU.
3016	*
3017	* @remarks No-long-jump zone!!!
3018	*/
3019	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3020	{
3021	NOREF(pVM);
3022
3023	AssertPtr(pVCpu);
3024	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3025
3026	int rc = VINF_SUCCESS;
3027	#if HC_ARCH_BITS == 64
3028	if (pVM->hm.s.fAllow64BitGuests)
3029	hmR0VmxLazySaveHostMsrs(pVCpu);
3030	#endif
3031
3032	/*
3033	* Host Sysenter MSRs.
3034	*/
3035	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3036	AssertRCReturn(rc, rc);
3037	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3038	if (HMVMX_IS_64BIT_HOST_MODE())
3039	{
3040	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3041	AssertRCReturn(rc, rc);
3042	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3043	}
3044	else
3045	{
3046	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3047	AssertRCReturn(rc, rc);
3048	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3049	}
3050	#elif HC_ARCH_BITS == 32
3051	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3052	AssertRCReturn(rc, rc);
3053	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3054	#else
3055	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3056	AssertRCReturn(rc, rc);
3057	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3058	#endif
3059	AssertRCReturn(rc, rc);
3060
3061	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3062	* hmR0VmxSetupExitCtls() !! */
3063	return rc;
3064	}
3065
3066
3067	/**
3068	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3069	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3070	* controls".
3071	*
3072	* @returns VBox status code.
3073	* @param pVCpu Pointer to the VMCPU.
3074	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3075	* out-of-sync. Make sure to update the required fields
3076	* before using them.
3077	*
3078	* @remarks No-long-jump zone!!!
3079	*/
3080	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3081	{
3082	int rc = VINF_SUCCESS;
3083	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3084	{
3085	PVM pVM = pVCpu->CTX_SUFF(pVM);
3086	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3087	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3088
3089	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3090	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3091
3092	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3093	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3094	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3095	else
3096	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3097
3098	/*
3099	* The following should -not- be set (since we're not in SMM mode):
3100	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3101	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3102	*/
3103
3104	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3105	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3106	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3107
3108	if ((val & zap) != val)
3109	{
3110	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3111	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3112	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3113	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3114	}
3115
3116	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3117	AssertRCReturn(rc, rc);
3118
3119	/* Update VCPU with the currently set VM-exit controls. */
3120	pVCpu->hm.s.vmx.u32EntryCtls = val;
3121	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3122	}
3123	return rc;
3124	}
3125
3126
3127	/**
3128	* Sets up the VM-exit controls in the VMCS.
3129	*
3130	* @returns VBox status code.
3131	* @param pVM Pointer to the VM.
3132	* @param pVCpu Pointer to the VMCPU.
3133	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3134	* out-of-sync. Make sure to update the required fields
3135	* before using them.
3136	*
3137	* @remarks requires EFER.
3138	*/
3139	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3140	{
3141	NOREF(pMixedCtx);
3142
3143	int rc = VINF_SUCCESS;
3144	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3145	{
3146	PVM pVM = pVCpu->CTX_SUFF(pVM);
3147	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3148	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3149
3150	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3151	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3152
3153	/*
3154	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3155	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3156	*/
3157	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3158	if (HMVMX_IS_64BIT_HOST_MODE())
3159	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3160	else
3161	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3162	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3163	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3164	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3165	else
3166	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3167	#endif
3168
3169	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3170	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3171
3172	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3173	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3174	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3175	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3176	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3177
3178	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3179	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3180
3181	if ((val & zap) != val)
3182	{
3183	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3184	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3185	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3186	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3187	}
3188
3189	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3190	AssertRCReturn(rc, rc);
3191
3192	/* Update VCPU with the currently set VM-exit controls. */
3193	pVCpu->hm.s.vmx.u32ExitCtls = val;
3194	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3195	}
3196	return rc;
3197	}
3198
3199
3200	/**
3201	* Loads the guest APIC and related state.
3202	*
3203	* @returns VBox status code.
3204	* @param pVM Pointer to the VM.
3205	* @param pVCpu Pointer to the VMCPU.
3206	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3207	* out-of-sync. Make sure to update the required fields
3208	* before using them.
3209	*/
3210	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3211	{
3212	NOREF(pMixedCtx);
3213
3214	int rc = VINF_SUCCESS;
3215	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3216	{
3217	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3218	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3219	{
3220	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3221
3222	bool fPendingIntr = false;
3223	uint8_t u8Tpr = 0;
3224	uint8_t u8PendingIntr = 0;
3225	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3226	AssertRCReturn(rc, rc);
3227
3228	/*
3229	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3230	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3231	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3232	* the interrupt when we VM-exit for other reasons.
3233	*/
3234	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3235	uint32_t u32TprThreshold = 0;
3236	if (fPendingIntr)
3237	{
3238	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3239	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3240	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3241	if (u8PendingPriority <= u8TprPriority)
3242	u32TprThreshold = u8PendingPriority;
3243	else
3244	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3245	}
3246	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3247
3248	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3249	AssertRCReturn(rc, rc);
3250	}
3251
3252	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3253	}
3254	return rc;
3255	}
3256
3257
3258	/**
3259	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3260	*
3261	* @returns Guest's interruptibility-state.
3262	* @param pVCpu Pointer to the VMCPU.
3263	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3264	* out-of-sync. Make sure to update the required fields
3265	* before using them.
3266	*
3267	* @remarks No-long-jump zone!!!
3268	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3269	*/
3270	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3271	{
3272	/*
3273	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3274	* inhibit interrupts or clear any existing interrupt-inhibition.
3275	*/
3276	uint32_t uIntrState = 0;
3277	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3278	{
3279	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3280	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3281	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3282	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3283	{
3284	/*
3285	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3286	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3287	*/
3288	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3289	}
3290	else if (pMixedCtx->eflags.Bits.u1IF)
3291	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3292	else
3293	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3294	}
3295	return uIntrState;
3296	}
3297
3298
3299	/**
3300	* Loads the guest's interruptibility-state into the guest-state area in the
3301	* VMCS.
3302	*
3303	* @returns VBox status code.
3304	* @param pVCpu Pointer to the VMCPU.
3305	* @param uIntrState The interruptibility-state to set.
3306	*/
3307	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3308	{
3309	NOREF(pVCpu);
3310	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3311	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3312	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3313	AssertRCReturn(rc, rc);
3314	return rc;
3315	}
3316
3317
3318	/**
3319	* Loads the guest's RIP into the guest-state area in the VMCS.
3320	*
3321	* @returns VBox status code.
3322	* @param pVCpu Pointer to the VMCPU.
3323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3324	* out-of-sync. Make sure to update the required fields
3325	* before using them.
3326	*
3327	* @remarks No-long-jump zone!!!
3328	*/
3329	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3330	{
3331	int rc = VINF_SUCCESS;
3332	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3333	{
3334	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3335	AssertRCReturn(rc, rc);
3336
3337	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3338	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3339	}
3340	return rc;
3341	}
3342
3343
3344	/**
3345	* Loads the guest's RSP into the guest-state area in the VMCS.
3346	*
3347	* @returns VBox status code.
3348	* @param pVCpu Pointer to the VMCPU.
3349	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3350	* out-of-sync. Make sure to update the required fields
3351	* before using them.
3352	*
3353	* @remarks No-long-jump zone!!!
3354	*/
3355	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3356	{
3357	int rc = VINF_SUCCESS;
3358	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3359	{
3360	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3361	AssertRCReturn(rc, rc);
3362
3363	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3364	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3365	}
3366	return rc;
3367	}
3368
3369
3370	/**
3371	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3372	*
3373	* @returns VBox status code.
3374	* @param pVCpu Pointer to the VMCPU.
3375	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3376	* out-of-sync. Make sure to update the required fields
3377	* before using them.
3378	*
3379	* @remarks No-long-jump zone!!!
3380	*/
3381	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3382	{
3383	int rc = VINF_SUCCESS;
3384	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3385	{
3386	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3387	Let us assert it as such and use 32-bit VMWRITE. */
3388	Assert(!(pMixedCtx->rflags.u64 >> 32));
3389	X86EFLAGS Eflags = pMixedCtx->eflags;
3390	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3391	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3392
3393	/*
3394	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3395	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3396	*/
3397	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3398	{
3399	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3400	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3401	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3402	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3403	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3404	}
3405
3406	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3407	AssertRCReturn(rc, rc);
3408
3409	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3410	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3411	}
3412	return rc;
3413	}
3414
3415
3416	/**
3417	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3418	*
3419	* @returns VBox status code.
3420	* @param pVCpu Pointer to the VMCPU.
3421	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3422	* out-of-sync. Make sure to update the required fields
3423	* before using them.
3424	*
3425	* @remarks No-long-jump zone!!!
3426	*/
3427	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3428	{
3429	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3430	AssertRCReturn(rc, rc);
3431	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3432	AssertRCReturn(rc, rc);
3433	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3434	AssertRCReturn(rc, rc);
3435	return rc;
3436	}
3437
3438
3439	/**
3440	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3441	* CR0 is partially shared with the host and we have to consider the FPU bits.
3442	*
3443	* @returns VBox status code.
3444	* @param pVM Pointer to the VM.
3445	* @param pVCpu Pointer to the VMCPU.
3446	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3447	* out-of-sync. Make sure to update the required fields
3448	* before using them.
3449	*
3450	* @remarks No-long-jump zone!!!
3451	*/
3452	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3453	{
3454	/*
3455	* Guest CR0.
3456	* Guest FPU.
3457	*/
3458	int rc = VINF_SUCCESS;
3459	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3460	{
3461	Assert(!(pMixedCtx->cr0 >> 32));
3462	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3463	PVM pVM = pVCpu->CTX_SUFF(pVM);
3464
3465	/* The guest's view (read access) of its CR0 is unblemished. */
3466	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3467	AssertRCReturn(rc, rc);
3468	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3469
3470	/* Setup VT-x's view of the guest CR0. */
3471	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3472	if (pVM->hm.s.fNestedPaging)
3473	{
3474	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3475	{
3476	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3477	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3478	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3479	}
3480	else
3481	{
3482	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3483	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3484	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3485	}
3486
3487	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3488	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3489	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3490
3491	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3492	AssertRCReturn(rc, rc);
3493	}
3494	else
3495	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3496
3497	/*
3498	* Guest FPU bits.
3499	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3500	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3501	*/
3502	u32GuestCR0 \|= X86_CR0_NE;
3503	bool fInterceptNM = false;
3504	if (CPUMIsGuestFPUStateActive(pVCpu))
3505	{
3506	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3507	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3508	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3509	}
3510	else
3511	{
3512	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3513	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3514	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3515	}
3516
3517	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3518	bool fInterceptMF = false;
3519	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3520	fInterceptMF = true;
3521
3522	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3523	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3524	{
3525	Assert(PDMVmmDevHeapIsEnabled(pVM));
3526	Assert(pVM->hm.s.vmx.pRealModeTSS);
3527	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3528	fInterceptNM = true;
3529	fInterceptMF = true;
3530	}
3531	else
3532	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3533
3534	if (fInterceptNM)
3535	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3536	else
3537	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3538
3539	if (fInterceptMF)
3540	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3541	else
3542	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3543
3544	/* Additional intercepts for debugging, define these yourself explicitly. */
3545	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3546	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3547	\| RT_BIT(X86_XCPT_BP)
3548	\| RT_BIT(X86_XCPT_DB)
3549	\| RT_BIT(X86_XCPT_DE)
3550	\| RT_BIT(X86_XCPT_NM)
3551	\| RT_BIT(X86_XCPT_UD)
3552	\| RT_BIT(X86_XCPT_NP)
3553	\| RT_BIT(X86_XCPT_SS)
3554	\| RT_BIT(X86_XCPT_GP)
3555	\| RT_BIT(X86_XCPT_PF)
3556	\| RT_BIT(X86_XCPT_MF)
3557	;
3558	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3559	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3560	#endif
3561
3562	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3563
3564	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3565	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3566	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3567	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3568	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3569	else
3570	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3571
3572	u32GuestCR0 \|= uSetCR0;
3573	u32GuestCR0 &= uZapCR0;
3574	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3575
3576	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3577	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3578	AssertRCReturn(rc, rc);
3579	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3580	AssertRCReturn(rc, rc);
3581	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3582
3583	/*
3584	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3585	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3586	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3587	*/
3588	uint32_t u32CR0Mask = 0;
3589	u32CR0Mask = X86_CR0_PE
3590	\| X86_CR0_NE
3591	\| X86_CR0_WP
3592	\| X86_CR0_PG
3593	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3594	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3595	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3596
3597	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3598	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3599	* and @bugref{6944}. */
3600	#if 0
3601	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3602	u32CR0Mask &= ~X86_CR0_PE;
3603	#endif
3604	if (pVM->hm.s.fNestedPaging)
3605	u32CR0Mask &= ~X86_CR0_WP;
3606
3607	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3608	if (fInterceptNM)
3609	{
3610	u32CR0Mask \|= X86_CR0_TS
3611	\| X86_CR0_MP;
3612	}
3613
3614	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3615	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3616	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3617	AssertRCReturn(rc, rc);
3618	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3619
3620	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3621	}
3622	return rc;
3623	}
3624
3625
3626	/**
3627	* Loads the guest control registers (CR3, CR4) into the guest-state area
3628	* in the VMCS.
3629	*
3630	* @returns VBox status code.
3631	* @param pVM Pointer to the VM.
3632	* @param pVCpu Pointer to the VMCPU.
3633	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3634	* out-of-sync. Make sure to update the required fields
3635	* before using them.
3636	*
3637	* @remarks No-long-jump zone!!!
3638	*/
3639	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3640	{
3641	int rc = VINF_SUCCESS;
3642	PVM pVM = pVCpu->CTX_SUFF(pVM);
3643
3644	/*
3645	* Guest CR2.
3646	* It's always loaded in the assembler code. Nothing to do here.
3647	*/
3648
3649	/*
3650	* Guest CR3.
3651	*/
3652	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3653	{
3654	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3655	if (pVM->hm.s.fNestedPaging)
3656	{
3657	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3658
3659	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3660	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3661	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3662	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3663
3664	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3665	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3666	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3667
3668	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3669	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3670	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3671	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3672
3673	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3674	AssertRCReturn(rc, rc);
3675	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3676
3677	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3678	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3679	{
3680	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3681	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3682	{
3683	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3684	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3685	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3686	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3687	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3688	}
3689
3690	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3691	have Unrestricted Execution to handle the guest when it's not using paging. */
3692	GCPhysGuestCR3 = pMixedCtx->cr3;
3693	}
3694	else
3695	{
3696	/*
3697	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3698	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3699	* EPT takes care of translating it to host-physical addresses.
3700	*/
3701	RTGCPHYS GCPhys;
3702	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3703	Assert(PDMVmmDevHeapIsEnabled(pVM));
3704
3705	/* We obtain it here every time as the guest could have relocated this PCI region. */
3706	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3707	AssertRCReturn(rc, rc);
3708
3709	GCPhysGuestCR3 = GCPhys;
3710	}
3711
3712	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3713	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3714	}
3715	else
3716	{
3717	/* Non-nested paging case, just use the hypervisor's CR3. */
3718	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3719
3720	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3721	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3722	}
3723	AssertRCReturn(rc, rc);
3724
3725	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3726	}
3727
3728	/*
3729	* Guest CR4.
3730	*/
3731	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3732	{
3733	Assert(!(pMixedCtx->cr4 >> 32));
3734	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3735
3736	/* The guest's view of its CR4 is unblemished. */
3737	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3738	AssertRCReturn(rc, rc);
3739	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3740
3741	/* Setup VT-x's view of the guest CR4. */
3742	/*
3743	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3744	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3745	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3746	*/
3747	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3748	{
3749	Assert(pVM->hm.s.vmx.pRealModeTSS);
3750	Assert(PDMVmmDevHeapIsEnabled(pVM));
3751	u32GuestCR4 &= ~X86_CR4_VME;
3752	}
3753
3754	if (pVM->hm.s.fNestedPaging)
3755	{
3756	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3757	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3758	{
3759	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3760	u32GuestCR4 \|= X86_CR4_PSE;
3761	/* Our identity mapping is a 32-bit page directory. */
3762	u32GuestCR4 &= ~X86_CR4_PAE;
3763	}
3764	/* else use guest CR4.*/
3765	}
3766	else
3767	{
3768	/*
3769	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3770	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3771	*/
3772	switch (pVCpu->hm.s.enmShadowMode)
3773	{
3774	case PGMMODE_REAL: /* Real-mode. */
3775	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3776	case PGMMODE_32_BIT: /* 32-bit paging. */
3777	{
3778	u32GuestCR4 &= ~X86_CR4_PAE;
3779	break;
3780	}
3781
3782	case PGMMODE_PAE: /* PAE paging. */
3783	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3784	{
3785	u32GuestCR4 \|= X86_CR4_PAE;
3786	break;
3787	}
3788
3789	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3790	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3791	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3792	break;
3793	#endif
3794	default:
3795	AssertFailed();
3796	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3797	}
3798	}
3799
3800	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3801	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3802	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3803	u32GuestCR4 \|= uSetCR4;
3804	u32GuestCR4 &= uZapCR4;
3805
3806	/* Write VT-x's view of the guest CR4 into the VMCS. */
3807	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3808	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3809	AssertRCReturn(rc, rc);
3810
3811	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3812	uint32_t u32CR4Mask = 0;
3813	u32CR4Mask = X86_CR4_VME
3814	\| X86_CR4_PAE
3815	\| X86_CR4_PGE
3816	\| X86_CR4_PSE
3817	\| X86_CR4_VMXE;
3818	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3819	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3820	AssertRCReturn(rc, rc);
3821
3822	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3823	}
3824	return rc;
3825	}
3826
3827
3828	/**
3829	* Loads the guest debug registers into the guest-state area in the VMCS.
3830	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3831	*
3832	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3833	*
3834	* @returns VBox status code.
3835	* @param pVCpu Pointer to the VMCPU.
3836	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3837	* out-of-sync. Make sure to update the required fields
3838	* before using them.
3839	*
3840	* @remarks No-long-jump zone!!!
3841	*/
3842	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3843	{
3844	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3845	return VINF_SUCCESS;
3846
3847	#ifdef VBOX_STRICT
3848	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3849	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3850	{
3851	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3852	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3853	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3854	}
3855	#endif
3856
3857	int rc;
3858	PVM pVM = pVCpu->CTX_SUFF(pVM);
3859	bool fInterceptDB = false;
3860	bool fInterceptMovDRx = false;
3861	if ( pVCpu->hm.s.fSingleInstruction
3862	\|\| DBGFIsStepping(pVCpu))
3863	{
3864	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3865	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3866	{
3867	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3868	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3869	AssertRCReturn(rc, rc);
3870	Assert(fInterceptDB == false);
3871	}
3872	else
3873	{
3874	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3875	pVCpu->hm.s.fClearTrapFlag = true;
3876	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3877	fInterceptDB = true;
3878	}
3879	}
3880
3881	if ( fInterceptDB
3882	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3883	{
3884	/*
3885	* Use the combined guest and host DRx values found in the hypervisor
3886	* register set because the debugger has breakpoints active or someone
3887	* is single stepping on the host side without a monitor trap flag.
3888	*
3889	* Note! DBGF expects a clean DR6 state before executing guest code.
3890	*/
3891	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3892	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3893	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3894	{
3895	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3896	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3897	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3898	}
3899	else
3900	#endif
3901	if (!CPUMIsHyperDebugStateActive(pVCpu))
3902	{
3903	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3904	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3905	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3906	}
3907
3908	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3909	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3910	AssertRCReturn(rc, rc);
3911
3912	pVCpu->hm.s.fUsingHyperDR7 = true;
3913	fInterceptDB = true;
3914	fInterceptMovDRx = true;
3915	}
3916	else
3917	{
3918	/*
3919	* If the guest has enabled debug registers, we need to load them prior to
3920	* executing guest code so they'll trigger at the right time.
3921	*/
3922	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3923	{
3924	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3925	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3926	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3927	{
3928	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3929	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3930	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3931	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3932	}
3933	else
3934	#endif
3935	if (!CPUMIsGuestDebugStateActive(pVCpu))
3936	{
3937	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3938	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3939	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3940	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3941	}
3942	Assert(!fInterceptDB);
3943	Assert(!fInterceptMovDRx);
3944	}
3945	/*
3946	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3947	* must intercept #DB in order to maintain a correct DR6 guest value.
3948	*/
3949	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3950	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3951	&& !CPUMIsGuestDebugStateActive(pVCpu))
3952	#else
3953	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3954	#endif
3955	{
3956	fInterceptMovDRx = true;
3957	fInterceptDB = true;
3958	}
3959
3960	/* Update guest DR7. */
3961	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3962	AssertRCReturn(rc, rc);
3963
3964	pVCpu->hm.s.fUsingHyperDR7 = false;
3965	}
3966
3967	/*
3968	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3969	*/
3970	if (fInterceptDB)
3971	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3972	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3973	{
3974	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3975	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3976	#endif
3977	}
3978	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3979	AssertRCReturn(rc, rc);
3980
3981	/*
3982	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3983	*/
3984	if (fInterceptMovDRx)
3985	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3986	else
3987	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3988	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3989	AssertRCReturn(rc, rc);
3990
3991	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3992	return VINF_SUCCESS;
3993	}
3994
3995
3996	#ifdef VBOX_STRICT
3997	/**
3998	* Strict function to validate segment registers.
3999	*
4000	* @remarks ASSUMES CR0 is up to date.
4001	*/
4002	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4003	{
4004	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4005	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4006	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4007	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4008	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4009	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4010	{
4011	/* Protected mode checks */
4012	/* CS */
4013	Assert(pCtx->cs.Attr.n.u1Present);
4014	Assert(!(pCtx->cs.Attr.u & 0xf00));
4015	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4016	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4017	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4018	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4019	\|\| (pCtx->cs.Attr.n.u1Granularity));
4020	/* CS cannot be loaded with NULL in protected mode. */
4021	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4022	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4023	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4024	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4025	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4026	else
4027	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4028	/* SS */
4029	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4030	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4031	if ( !(pCtx->cr0 & X86_CR0_PE)
4032	\|\| pCtx->cs.Attr.n.u4Type == 3)
4033	{
4034	Assert(!pCtx->ss.Attr.n.u2Dpl);
4035	}
4036	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4037	{
4038	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4039	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4040	Assert(pCtx->ss.Attr.n.u1Present);
4041	Assert(!(pCtx->ss.Attr.u & 0xf00));
4042	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4043	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4044	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4045	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4046	\|\| (pCtx->ss.Attr.n.u1Granularity));
4047	}
4048	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4049	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4050	{
4051	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4052	Assert(pCtx->ds.Attr.n.u1Present);
4053	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4054	Assert(!(pCtx->ds.Attr.u & 0xf00));
4055	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4056	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4057	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4058	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4059	\|\| (pCtx->ds.Attr.n.u1Granularity));
4060	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4061	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4062	}
4063	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4064	{
4065	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4066	Assert(pCtx->es.Attr.n.u1Present);
4067	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4068	Assert(!(pCtx->es.Attr.u & 0xf00));
4069	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4070	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4071	\|\| !(pCtx->es.Attr.n.u1Granularity));
4072	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4073	\|\| (pCtx->es.Attr.n.u1Granularity));
4074	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4075	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4076	}
4077	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4078	{
4079	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4080	Assert(pCtx->fs.Attr.n.u1Present);
4081	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4082	Assert(!(pCtx->fs.Attr.u & 0xf00));
4083	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4084	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4085	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4086	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4087	\|\| (pCtx->fs.Attr.n.u1Granularity));
4088	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4089	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4090	}
4091	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4092	{
4093	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4094	Assert(pCtx->gs.Attr.n.u1Present);
4095	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4096	Assert(!(pCtx->gs.Attr.u & 0xf00));
4097	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4098	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4099	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4100	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4101	\|\| (pCtx->gs.Attr.n.u1Granularity));
4102	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4103	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4104	}
4105	/* 64-bit capable CPUs. */
4106	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4107	if (HMVMX_IS_64BIT_HOST_MODE())
4108	{
4109	Assert(!(pCtx->cs.u64Base >> 32));
4110	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4111	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4112	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4113	}
4114	# endif
4115	}
4116	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4117	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4118	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4119	{
4120	/* Real and v86 mode checks. */
4121	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4122	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4123	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4124	{
4125	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4126	}
4127	else
4128	{
4129	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4130	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4131	}
4132
4133	/* CS */
4134	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4135	Assert(pCtx->cs.u32Limit == 0xffff);
4136	Assert(u32CSAttr == 0xf3);
4137	/* SS */
4138	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4139	Assert(pCtx->ss.u32Limit == 0xffff);
4140	Assert(u32SSAttr == 0xf3);
4141	/* DS */
4142	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4143	Assert(pCtx->ds.u32Limit == 0xffff);
4144	Assert(u32DSAttr == 0xf3);
4145	/* ES */
4146	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4147	Assert(pCtx->es.u32Limit == 0xffff);
4148	Assert(u32ESAttr == 0xf3);
4149	/* FS */
4150	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4151	Assert(pCtx->fs.u32Limit == 0xffff);
4152	Assert(u32FSAttr == 0xf3);
4153	/* GS */
4154	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4155	Assert(pCtx->gs.u32Limit == 0xffff);
4156	Assert(u32GSAttr == 0xf3);
4157	/* 64-bit capable CPUs. */
4158	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4159	if (HMVMX_IS_64BIT_HOST_MODE())
4160	{
4161	Assert(!(pCtx->cs.u64Base >> 32));
4162	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4163	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4164	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4165	}
4166	# endif
4167	}
4168	}
4169	#endif /* VBOX_STRICT */
4170
4171
4172	/**
4173	* Writes a guest segment register into the guest-state area in the VMCS.
4174	*
4175	* @returns VBox status code.
4176	* @param pVCpu Pointer to the VMCPU.
4177	* @param idxSel Index of the selector in the VMCS.
4178	* @param idxLimit Index of the segment limit in the VMCS.
4179	* @param idxBase Index of the segment base in the VMCS.
4180	* @param idxAccess Index of the access rights of the segment in the VMCS.
4181	* @param pSelReg Pointer to the segment selector.
4182	*
4183	* @remarks No-long-jump zone!!!
4184	*/
4185	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4186	uint32_t idxAccess, PCPUMSELREG pSelReg)
4187	{
4188	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4189	AssertRCReturn(rc, rc);
4190	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4191	AssertRCReturn(rc, rc);
4192	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4193	AssertRCReturn(rc, rc);
4194
4195	uint32_t u32Access = pSelReg->Attr.u;
4196	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4197	{
4198	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4199	u32Access = 0xf3;
4200	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4201	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4202	}
4203	else
4204	{
4205	/*
4206	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4207	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4208	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4209	* loaded in protected-mode have their attribute as 0.
4210	*/
4211	if (!u32Access)
4212	u32Access = X86DESCATTR_UNUSABLE;
4213	}
4214
4215	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4216	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4217	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4218
4219	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4220	AssertRCReturn(rc, rc);
4221	return rc;
4222	}
4223
4224
4225	/**
4226	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4227	* into the guest-state area in the VMCS.
4228	*
4229	* @returns VBox status code.
4230	* @param pVM Pointer to the VM.
4231	* @param pVCPU Pointer to the VMCPU.
4232	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4233	* out-of-sync. Make sure to update the required fields
4234	* before using them.
4235	*
4236	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4237	* @remarks No-long-jump zone!!!
4238	*/
4239	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4240	{
4241	int rc = VERR_INTERNAL_ERROR_5;
4242	PVM pVM = pVCpu->CTX_SUFF(pVM);
4243
4244	/*
4245	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4246	*/
4247	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4248	{
4249	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4250	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4251	{
4252	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4253	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4254	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4255	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4256	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4257	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4258	}
4259
4260	#ifdef VBOX_WITH_REM
4261	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4262	{
4263	Assert(pVM->hm.s.vmx.pRealModeTSS);
4264	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4265	if ( pVCpu->hm.s.vmx.fWasInRealMode
4266	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4267	{
4268	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4269	in real-mode (e.g. OpenBSD 4.0) */
4270	REMFlushTBs(pVM);
4271	Log4(("Load: Switch to protected mode detected!\n"));
4272	pVCpu->hm.s.vmx.fWasInRealMode = false;
4273	}
4274	}
4275	#endif
4276	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4277	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4278	AssertRCReturn(rc, rc);
4279	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4280	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4281	AssertRCReturn(rc, rc);
4282	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4283	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4284	AssertRCReturn(rc, rc);
4285	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4286	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4287	AssertRCReturn(rc, rc);
4288	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4289	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4290	AssertRCReturn(rc, rc);
4291	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4292	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4293	AssertRCReturn(rc, rc);
4294
4295	#ifdef VBOX_STRICT
4296	/* Validate. */
4297	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4298	#endif
4299
4300	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4301	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4302	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4303	}
4304
4305	/*
4306	* Guest TR.
4307	*/
4308	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4309	{
4310	/*
4311	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4312	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4313	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4314	*/
4315	uint16_t u16Sel = 0;
4316	uint32_t u32Limit = 0;
4317	uint64_t u64Base = 0;
4318	uint32_t u32AccessRights = 0;
4319
4320	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4321	{
4322	u16Sel = pMixedCtx->tr.Sel;
4323	u32Limit = pMixedCtx->tr.u32Limit;
4324	u64Base = pMixedCtx->tr.u64Base;
4325	u32AccessRights = pMixedCtx->tr.Attr.u;
4326	}
4327	else
4328	{
4329	Assert(pVM->hm.s.vmx.pRealModeTSS);
4330	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4331
4332	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4333	RTGCPHYS GCPhys;
4334	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4335	AssertRCReturn(rc, rc);
4336
4337	X86DESCATTR DescAttr;
4338	DescAttr.u = 0;
4339	DescAttr.n.u1Present = 1;
4340	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4341
4342	u16Sel = 0;
4343	u32Limit = HM_VTX_TSS_SIZE;
4344	u64Base = GCPhys; /* in real-mode phys = virt. */
4345	u32AccessRights = DescAttr.u;
4346	}
4347
4348	/* Validate. */
4349	Assert(!(u16Sel & RT_BIT(2)));
4350	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4351	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4352	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4353	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4354	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4355	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4356	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4357	Assert( (u32Limit & 0xfff) == 0xfff
4358	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4359	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4360	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4361
4362	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4363	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4364	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4365	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4366
4367	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4368	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4369	}
4370
4371	/*
4372	* Guest GDTR.
4373	*/
4374	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4375	{
4376	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4377	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4378
4379	/* Validate. */
4380	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4381
4382	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4383	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4384	}
4385
4386	/*
4387	* Guest LDTR.
4388	*/
4389	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4390	{
4391	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4392	uint32_t u32Access = 0;
4393	if (!pMixedCtx->ldtr.Attr.u)
4394	u32Access = X86DESCATTR_UNUSABLE;
4395	else
4396	u32Access = pMixedCtx->ldtr.Attr.u;
4397
4398	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4399	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4400	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4401	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4402
4403	/* Validate. */
4404	if (!(u32Access & X86DESCATTR_UNUSABLE))
4405	{
4406	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4407	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4408	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4409	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4410	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4411	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4412	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4413	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4414	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4415	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4416	}
4417
4418	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4419	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4420	}
4421
4422	/*
4423	* Guest IDTR.
4424	*/
4425	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4426	{
4427	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4428	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4429
4430	/* Validate. */
4431	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4432
4433	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4434	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4435	}
4436
4437	return VINF_SUCCESS;
4438	}
4439
4440
4441	/**
4442	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4443	* areas. These MSRs will automatically be loaded to the host CPU on every
4444	* successful VM entry and stored from the host CPU on every successful VM-exit.
4445	*
4446	* This also creates/updates MSR slots for the host MSRs. The actual host
4447	* MSR values are -not- updated here for performance reasons. See
4448	* hmR0VmxSaveHostMsrs().
4449	*
4450	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4451	*
4452	* @returns VBox status code.
4453	* @param pVCpu Pointer to the VMCPU.
4454	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4455	* out-of-sync. Make sure to update the required fields
4456	* before using them.
4457	*
4458	* @remarks No-long-jump zone!!!
4459	*/
4460	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4461	{
4462	AssertPtr(pVCpu);
4463	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4464
4465	/*
4466	* MSRs that we use the auto-load/store MSR area in the VMCS.
4467	*/
4468	PVM pVM = pVCpu->CTX_SUFF(pVM);
4469	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4470	{
4471	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4472	if (pVM->hm.s.fAllow64BitGuests)
4473	{
4474	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4475	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4476	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4477	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4478	# ifdef DEBUG
4479	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4480	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4481	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4482	# endif
4483	}
4484	#endif
4485	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4486	}
4487
4488	/*
4489	* Guest Sysenter MSRs.
4490	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4491	* VM-exits on WRMSRs for these MSRs.
4492	*/
4493	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4494	{
4495	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4496	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4497	}
4498
4499	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4500	{
4501	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4502	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4503	}
4504
4505	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4506	{
4507	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4508	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4509	}
4510
4511	return VINF_SUCCESS;
4512	}
4513
4514
4515	/**
4516	* Loads the guest activity state into the guest-state area in the VMCS.
4517	*
4518	* @returns VBox status code.
4519	* @param pVCpu Pointer to the VMCPU.
4520	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4521	* out-of-sync. Make sure to update the required fields
4522	* before using them.
4523	*
4524	* @remarks No-long-jump zone!!!
4525	*/
4526	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4527	{
4528	NOREF(pCtx);
4529	/** @todo See if we can make use of other states, e.g.
4530	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4531	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4532	{
4533	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4534	AssertRCReturn(rc, rc);
4535
4536	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4537	}
4538	return VINF_SUCCESS;
4539	}
4540
4541
4542	/**
4543	* Sets up the appropriate function to run guest code.
4544	*
4545	* @returns VBox status code.
4546	* @param pVCpu Pointer to the VMCPU.
4547	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4548	* out-of-sync. Make sure to update the required fields
4549	* before using them.
4550	*
4551	* @remarks No-long-jump zone!!!
4552	*/
4553	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4554	{
4555	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4556	{
4557	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4558	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4559	#endif
4560	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4561	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4562	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4563	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4564	{
4565	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4566	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4567	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS));
4568	}
4569	#else
4570	/* 64-bit host or hybrid host. */
4571	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4572	#endif
4573	}
4574	else
4575	{
4576	/* Guest is not in long mode, use the 32-bit handler. */
4577	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4578	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4579	{
4580	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4581	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4582	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS));
4583	}
4584	#else
4585	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4586	#endif
4587	}
4588	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4589	return VINF_SUCCESS;
4590	}
4591
4592
4593	/**
4594	* Wrapper for running the guest code in VT-x.
4595	*
4596	* @returns VBox strict status code.
4597	* @param pVM Pointer to the VM.
4598	* @param pVCpu Pointer to the VMCPU.
4599	* @param pCtx Pointer to the guest-CPU context.
4600	*
4601	* @remarks No-long-jump zone!!!
4602	*/
4603	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4604	{
4605	/*
4606	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4607	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4608	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4609	*/
4610	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4611	/** @todo Add stats for resume vs launch. */
4612	#ifdef VBOX_WITH_KERNEL_USING_XMM
4613	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4614	#else
4615	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4616	#endif
4617	}
4618
4619
4620	/**
4621	* Reports world-switch error and dumps some useful debug info.
4622	*
4623	* @param pVM Pointer to the VM.
4624	* @param pVCpu Pointer to the VMCPU.
4625	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4626	* @param pCtx Pointer to the guest-CPU context.
4627	* @param pVmxTransient Pointer to the VMX transient structure (only
4628	* exitReason updated).
4629	*/
4630	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4631	{
4632	Assert(pVM);
4633	Assert(pVCpu);
4634	Assert(pCtx);
4635	Assert(pVmxTransient);
4636	HMVMX_ASSERT_PREEMPT_SAFE();
4637
4638	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4639	switch (rcVMRun)
4640	{
4641	case VERR_VMX_INVALID_VMXON_PTR:
4642	AssertFailed();
4643	break;
4644	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4645	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4646	{
4647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4648	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4649	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4650	AssertRC(rc);
4651
4652	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4653	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4654	Cannot do it here as we may have been long preempted. */
4655
4656	#ifdef VBOX_STRICT
4657	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4658	pVmxTransient->uExitReason));
4659	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4660	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4661	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4662	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4663	else
4664	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4665	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4666	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4667
4668	/* VMX control bits. */
4669	uint32_t u32Val;
4670	uint64_t u64Val;
4671	HMVMXHCUINTREG uHCReg;
4672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4673	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4675	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4677	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4679	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4686	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4687	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4688	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4689	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4690	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4691	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4692	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4693	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4694	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4695	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4696	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4697	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4698	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4699	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4700	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4701	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4702	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4703	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4704	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4705	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4706	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4707	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4708	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4709	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4710	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4711	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4712	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4713	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4714
4715	/* Guest bits. */
4716	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4717	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4718	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4719	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4720	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4721	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4722	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4723	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4724
4725	/* Host bits. */
4726	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4727	Log4(("Host CR0 %#RHr\n", uHCReg));
4728	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4729	Log4(("Host CR3 %#RHr\n", uHCReg));
4730	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4731	Log4(("Host CR4 %#RHr\n", uHCReg));
4732
4733	RTGDTR HostGdtr;
4734	PCX86DESCHC pDesc;
4735	ASMGetGDTR(&HostGdtr);
4736	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4737	Log4(("Host CS %#08x\n", u32Val));
4738	if (u32Val < HostGdtr.cbGdt)
4739	{
4740	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4741	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4742	}
4743
4744	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4745	Log4(("Host DS %#08x\n", u32Val));
4746	if (u32Val < HostGdtr.cbGdt)
4747	{
4748	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4749	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4750	}
4751
4752	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4753	Log4(("Host ES %#08x\n", u32Val));
4754	if (u32Val < HostGdtr.cbGdt)
4755	{
4756	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4757	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4758	}
4759
4760	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4761	Log4(("Host FS %#08x\n", u32Val));
4762	if (u32Val < HostGdtr.cbGdt)
4763	{
4764	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4765	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4766	}
4767
4768	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4769	Log4(("Host GS %#08x\n", u32Val));
4770	if (u32Val < HostGdtr.cbGdt)
4771	{
4772	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4773	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4774	}
4775
4776	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4777	Log4(("Host SS %#08x\n", u32Val));
4778	if (u32Val < HostGdtr.cbGdt)
4779	{
4780	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4781	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4782	}
4783
4784	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4785	Log4(("Host TR %#08x\n", u32Val));
4786	if (u32Val < HostGdtr.cbGdt)
4787	{
4788	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4789	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4790	}
4791
4792	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4793	Log4(("Host TR Base %#RHv\n", uHCReg));
4794	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4795	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4796	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4797	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4798	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4799	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4800	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4801	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4802	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4803	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4804	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4805	Log4(("Host RSP %#RHv\n", uHCReg));
4806	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4807	Log4(("Host RIP %#RHv\n", uHCReg));
4808	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4809	if (HMVMX_IS_64BIT_HOST_MODE())
4810	{
4811	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4812	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4813	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4814	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4815	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4816	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4817	}
4818	# endif
4819	#endif /* VBOX_STRICT */
4820	break;
4821	}
4822
4823	default:
4824	/* Impossible */
4825	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4826	break;
4827	}
4828	NOREF(pVM); NOREF(pCtx);
4829	}
4830
4831
4832	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4833	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4834	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4835	#endif
4836	#ifdef VBOX_STRICT
4837	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4838	{
4839	switch (idxField)
4840	{
4841	case VMX_VMCS_GUEST_RIP:
4842	case VMX_VMCS_GUEST_RSP:
4843	case VMX_VMCS_GUEST_SYSENTER_EIP:
4844	case VMX_VMCS_GUEST_SYSENTER_ESP:
4845	case VMX_VMCS_GUEST_GDTR_BASE:
4846	case VMX_VMCS_GUEST_IDTR_BASE:
4847	case VMX_VMCS_GUEST_CS_BASE:
4848	case VMX_VMCS_GUEST_DS_BASE:
4849	case VMX_VMCS_GUEST_ES_BASE:
4850	case VMX_VMCS_GUEST_FS_BASE:
4851	case VMX_VMCS_GUEST_GS_BASE:
4852	case VMX_VMCS_GUEST_SS_BASE:
4853	case VMX_VMCS_GUEST_LDTR_BASE:
4854	case VMX_VMCS_GUEST_TR_BASE:
4855	case VMX_VMCS_GUEST_CR3:
4856	return true;
4857	}
4858	return false;
4859	}
4860
4861	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4862	{
4863	switch (idxField)
4864	{
4865	/* Read-only fields. */
4866	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4867	return true;
4868	}
4869	/* Remaining readable fields should also be writable. */
4870	return hmR0VmxIsValidWriteField(idxField);
4871	}
4872	#endif /* VBOX_STRICT */
4873
4874
4875	/**
4876	* Executes the specified handler in 64-bit mode.
4877	*
4878	* @returns VBox status code.
4879	* @param pVM Pointer to the VM.
4880	* @param pVCpu Pointer to the VMCPU.
4881	* @param pCtx Pointer to the guest CPU context.
4882	* @param enmOp The operation to perform.
4883	* @param cbParam Number of parameters.
4884	* @param paParam Array of 32-bit parameters.
4885	*/
4886	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4887	uint32_t *paParam)
4888	{
4889	int rc, rc2;
4890	PHMGLOBALCPUINFO pCpu;
4891	RTHCPHYS HCPhysCpuPage;
4892	RTCCUINTREG uOldEflags;
4893
4894	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4895	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4896	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4897	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4898
4899	#ifdef VBOX_STRICT
4900	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4901	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4902
4903	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4904	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4905	#endif
4906
4907	/* Disable interrupts. */
4908	uOldEflags = ASMIntDisableFlags();
4909
4910	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4911	RTCPUID idHostCpu = RTMpCpuId();
4912	CPUMR0SetLApic(pVCpu, idHostCpu);
4913	#endif
4914
4915	pCpu = HMR0GetCurrentCpu();
4916	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4917
4918	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4919	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4920
4921	/* Leave VMX Root Mode. */
4922	VMXDisable();
4923
4924	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4925
4926	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4927	CPUMSetHyperEIP(pVCpu, enmOp);
4928	for (int i = (int)cbParam - 1; i >= 0; i--)
4929	CPUMPushHyper(pVCpu, paParam[i]);
4930
4931	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4932
4933	/* Call the switcher. */
4934	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4935	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4936
4937	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4938	/* Make sure the VMX instructions don't cause #UD faults. */
4939	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4940
4941	/* Re-enter VMX Root Mode */
4942	rc2 = VMXEnable(HCPhysCpuPage);
4943	if (RT_FAILURE(rc2))
4944	{
4945	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4946	ASMSetFlags(uOldEflags);
4947	return rc2;
4948	}
4949
4950	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4951	AssertRC(rc2);
4952	Assert(!(ASMGetFlags() & X86_EFL_IF));
4953	ASMSetFlags(uOldEflags);
4954	return rc;
4955	}
4956
4957
4958	/**
4959	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4960	* supporting 64-bit guests.
4961	*
4962	* @returns VBox status code.
4963	* @param fResume Whether to VMLAUNCH or VMRESUME.
4964	* @param pCtx Pointer to the guest-CPU context.
4965	* @param pCache Pointer to the VMCS cache.
4966	* @param pVM Pointer to the VM.
4967	* @param pVCpu Pointer to the VMCPU.
4968	*/
4969	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4970	{
4971	uint32_t aParam[6];
4972	PHMGLOBALCPUINFO pCpu = NULL;
4973	RTHCPHYS HCPhysCpuPage = 0;
4974	int rc = VERR_INTERNAL_ERROR_5;
4975
4976	pCpu = HMR0GetCurrentCpu();
4977	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4978
4979	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4980	pCache->uPos = 1;
4981	pCache->interPD = PGMGetInterPaeCR3(pVM);
4982	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4983	#endif
4984
4985	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4986	pCache->TestIn.HCPhysCpuPage = 0;
4987	pCache->TestIn.HCPhysVmcs = 0;
4988	pCache->TestIn.pCache = 0;
4989	pCache->TestOut.HCPhysVmcs = 0;
4990	pCache->TestOut.pCache = 0;
4991	pCache->TestOut.pCtx = 0;
4992	pCache->TestOut.eflags = 0;
4993	#endif
4994
4995	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4996	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4997	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4998	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4999	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5000	aParam[5] = 0;
5001
5002	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5003	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5004	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5005	#endif
5006	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5007
5008	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5009	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5010	Assert(pCtx->dr[4] == 10);
5011	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5012	#endif
5013
5014	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5015	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5016	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5017	pVCpu->hm.s.vmx.HCPhysVmcs));
5018	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5019	pCache->TestOut.HCPhysVmcs));
5020	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5021	pCache->TestOut.pCache));
5022	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5023	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5024	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5025	pCache->TestOut.pCtx));
5026	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5027	#endif
5028	return rc;
5029	}
5030
5031
5032	/**
5033	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5034	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5035	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5036	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5037	*
5038	* @returns VBox status code.
5039	* @param pVM Pointer to the VM.
5040	* @param pVCpu Pointer to the VMCPU.
5041	*/
5042	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5043	{
5044	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5045	{ \
5046	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5047	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5048	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5049	++cReadFields; \
5050	}
5051
5052	AssertPtr(pVM);
5053	AssertPtr(pVCpu);
5054	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5055	uint32_t cReadFields = 0;
5056
5057	/*
5058	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5059	* and serve to indicate exceptions to the rules.
5060	*/
5061
5062	/* Guest-natural selector base fields. */
5063	#if 0
5064	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5066	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5067	#endif
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5069	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5070	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5071	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5080	#if 0
5081	/* Unused natural width guest-state fields. */
5082	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5083	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5084	#endif
5085	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5086	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5087
5088	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5089	#if 0
5090	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5091	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5092	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5093	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5094	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5095	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5096	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5097	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5098	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5099	#endif
5100
5101	/* Natural width guest-state fields. */
5102	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5103	#if 0
5104	/* Currently unused field. */
5105	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5106	#endif
5107
5108	if (pVM->hm.s.fNestedPaging)
5109	{
5110	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5111	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5112	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5113	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5114	}
5115	else
5116	{
5117	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5118	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5119	}
5120
5121	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5122	return VINF_SUCCESS;
5123	}
5124
5125
5126	/**
5127	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5128	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5129	* darwin, running 64-bit guests).
5130	*
5131	* @returns VBox status code.
5132	* @param pVCpu Pointer to the VMCPU.
5133	* @param idxField The VMCS field encoding.
5134	* @param u64Val 16, 32 or 64-bit value.
5135	*/
5136	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5137	{
5138	int rc;
5139	switch (idxField)
5140	{
5141	/*
5142	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5143	*/
5144	/* 64-bit Control fields. */
5145	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5146	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5147	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5148	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5149	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5150	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5151	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5152	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5153	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5154	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5155	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5156	case VMX_VMCS64_CTRL_EPTP_FULL:
5157	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5158	/* 64-bit Guest-state fields. */
5159	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5160	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5161	case VMX_VMCS64_GUEST_PAT_FULL:
5162	case VMX_VMCS64_GUEST_EFER_FULL:
5163	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5164	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5165	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5166	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5167	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5168	/* 64-bit Host-state fields. */
5169	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5170	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5171	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5172	{
5173	rc = VMXWriteVmcs32(idxField, u64Val);
5174	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5175	break;
5176	}
5177
5178	/*
5179	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5180	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5181	*/
5182	/* Natural-width Guest-state fields. */
5183	case VMX_VMCS_GUEST_CR3:
5184	case VMX_VMCS_GUEST_ES_BASE:
5185	case VMX_VMCS_GUEST_CS_BASE:
5186	case VMX_VMCS_GUEST_SS_BASE:
5187	case VMX_VMCS_GUEST_DS_BASE:
5188	case VMX_VMCS_GUEST_FS_BASE:
5189	case VMX_VMCS_GUEST_GS_BASE:
5190	case VMX_VMCS_GUEST_LDTR_BASE:
5191	case VMX_VMCS_GUEST_TR_BASE:
5192	case VMX_VMCS_GUEST_GDTR_BASE:
5193	case VMX_VMCS_GUEST_IDTR_BASE:
5194	case VMX_VMCS_GUEST_RSP:
5195	case VMX_VMCS_GUEST_RIP:
5196	case VMX_VMCS_GUEST_SYSENTER_ESP:
5197	case VMX_VMCS_GUEST_SYSENTER_EIP:
5198	{
5199	if (!(u64Val >> 32))
5200	{
5201	/* If this field is 64-bit, VT-x will zero out the top bits. */
5202	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5203	}
5204	else
5205	{
5206	/* Assert that only the 32->64 switcher case should ever come here. */
5207	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5208	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5209	}
5210	break;
5211	}
5212
5213	default:
5214	{
5215	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5216	rc = VERR_INVALID_PARAMETER;
5217	break;
5218	}
5219	}
5220	AssertRCReturn(rc, rc);
5221	return rc;
5222	}
5223
5224
5225	/**
5226	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5227	* hosts (except darwin) for 64-bit guests.
5228	*
5229	* @param pVCpu Pointer to the VMCPU.
5230	* @param idxField The VMCS field encoding.
5231	* @param u64Val 16, 32 or 64-bit value.
5232	*/
5233	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5234	{
5235	AssertPtr(pVCpu);
5236	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5237
5238	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5239	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5240
5241	/* Make sure there are no duplicates. */
5242	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5243	{
5244	if (pCache->Write.aField[i] == idxField)
5245	{
5246	pCache->Write.aFieldVal[i] = u64Val;
5247	return VINF_SUCCESS;
5248	}
5249	}
5250
5251	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5252	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5253	pCache->Write.cValidEntries++;
5254	return VINF_SUCCESS;
5255	}
5256
5257	/* Enable later when the assembly code uses these as callbacks. */
5258	#if 0
5259	/*
5260	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5261	*
5262	* @param pVCpu Pointer to the VMCPU.
5263	* @param pCache Pointer to the VMCS cache.
5264	*
5265	* @remarks No-long-jump zone!!!
5266	*/
5267	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5268	{
5269	AssertPtr(pCache);
5270	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5271	{
5272	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5273	AssertRC(rc);
5274	}
5275	pCache->Write.cValidEntries = 0;
5276	}
5277
5278
5279	/**
5280	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5281	*
5282	* @param pVCpu Pointer to the VMCPU.
5283	* @param pCache Pointer to the VMCS cache.
5284	*
5285	* @remarks No-long-jump zone!!!
5286	*/
5287	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5288	{
5289	AssertPtr(pCache);
5290	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5291	{
5292	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5293	AssertRC(rc);
5294	}
5295	}
5296	#endif
5297	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5298
5299
5300	/**
5301	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5302	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5303	* timer.
5304	*
5305	* @returns VBox status code.
5306	* @param pVCpu Pointer to the VMCPU.
5307	*
5308	* @remarks No-long-jump zone!!!
5309	*/
5310	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5311	{
5312	int rc = VERR_INTERNAL_ERROR_5;
5313	bool fOffsettedTsc = false;
5314	PVM pVM = pVCpu->CTX_SUFF(pVM);
5315	if (pVM->hm.s.vmx.fUsePreemptTimer)
5316	{
5317	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5318
5319	/* Make sure the returned values have sane upper and lower boundaries. */
5320	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5321	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5322	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5323	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5324
5325	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5326	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5327	}
5328	else
5329	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5330
5331	if (fOffsettedTsc)
5332	{
5333	uint64_t u64CurTSC = ASMReadTSC();
5334	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5335	{
5336	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5337	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5338
5339	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5340	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5341	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5342	}
5343	else
5344	{
5345	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5346	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5347	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5348	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5349	}
5350	}
5351	else
5352	{
5353	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5354	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5355	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5356	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5357	}
5358	}
5359
5360
5361	/**
5362	* Determines if an exception is a contributory exception. Contributory
5363	* exceptions are ones which can cause double-faults. Page-fault is
5364	* intentionally not included here as it's a conditional contributory exception.
5365	*
5366	* @returns true if the exception is contributory, false otherwise.
5367	* @param uVector The exception vector.
5368	*/
5369	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5370	{
5371	switch (uVector)
5372	{
5373	case X86_XCPT_GP:
5374	case X86_XCPT_SS:
5375	case X86_XCPT_NP:
5376	case X86_XCPT_TS:
5377	case X86_XCPT_DE:
5378	return true;
5379	default:
5380	break;
5381	}
5382	return false;
5383	}
5384
5385
5386	/**
5387	* Sets an event as a pending event to be injected into the guest.
5388	*
5389	* @param pVCpu Pointer to the VMCPU.
5390	* @param u32IntInfo The VM-entry interruption-information field.
5391	* @param cbInstr The VM-entry instruction length in bytes (for software
5392	* interrupts, exceptions and privileged software
5393	* exceptions).
5394	* @param u32ErrCode The VM-entry exception error code.
5395	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5396	* page-fault.
5397	*
5398	* @remarks Statistics counter assumes this is a guest event being injected or
5399	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5400	* always incremented.
5401	*/
5402	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5403	RTGCUINTPTR GCPtrFaultAddress)
5404	{
5405	Assert(!pVCpu->hm.s.Event.fPending);
5406	pVCpu->hm.s.Event.fPending = true;
5407	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5408	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5409	pVCpu->hm.s.Event.cbInstr = cbInstr;
5410	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5411
5412	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5413	}
5414
5415
5416	/**
5417	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5418	*
5419	* @param pVCpu Pointer to the VMCPU.
5420	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5421	* out-of-sync. Make sure to update the required fields
5422	* before using them.
5423	*/
5424	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5425	{
5426	NOREF(pMixedCtx);
5427	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5428	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5429	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5430	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5431	}
5432
5433
5434	/**
5435	* Handle a condition that occurred while delivering an event through the guest
5436	* IDT.
5437	*
5438	* @returns VBox status code (informational error codes included).
5439	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5440	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5441	* continue execution of the guest which will delivery the #DF.
5442	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5443	*
5444	* @param pVCpu Pointer to the VMCPU.
5445	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5446	* out-of-sync. Make sure to update the required fields
5447	* before using them.
5448	* @param pVmxTransient Pointer to the VMX transient structure.
5449	*
5450	* @remarks No-long-jump zone!!!
5451	*/
5452	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5453	{
5454	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5455	AssertRCReturn(rc, rc);
5456	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5457	{
5458	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5459	AssertRCReturn(rc, rc);
5460
5461	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5462	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5463	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5464
5465	typedef enum
5466	{
5467	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5468	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5469	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5470	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5471	} VMXREFLECTXCPT;
5472
5473	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5474	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5475	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5476	{
5477	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5478	{
5479	enmReflect = VMXREFLECTXCPT_XCPT;
5480	#ifdef VBOX_STRICT
5481	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5482	&& uExitVector == X86_XCPT_PF)
5483	{
5484	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5485	}
5486	#endif
5487	if ( uExitVector == X86_XCPT_PF
5488	&& uIdtVector == X86_XCPT_PF)
5489	{
5490	pVmxTransient->fVectoringPF = true;
5491	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5492	}
5493	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5494	&& hmR0VmxIsContributoryXcpt(uExitVector)
5495	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5496	\|\| uIdtVector == X86_XCPT_PF))
5497	{
5498	enmReflect = VMXREFLECTXCPT_DF;
5499	}
5500	else if (uIdtVector == X86_XCPT_DF)
5501	enmReflect = VMXREFLECTXCPT_TF;
5502	}
5503	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5504	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5505	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5506	{
5507	/*
5508	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5509	* (whatever they are) as they reoccur when restarting the instruction.
5510	*/
5511	enmReflect = VMXREFLECTXCPT_XCPT;
5512	}
5513	}
5514	else
5515	{
5516	/*
5517	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5518	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5519	* original exception to the guest after handling the VM-exit.
5520	*/
5521	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5522	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5523	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5524	{
5525	enmReflect = VMXREFLECTXCPT_XCPT;
5526	}
5527	}
5528
5529	switch (enmReflect)
5530	{
5531	case VMXREFLECTXCPT_XCPT:
5532	{
5533	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5534	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5535	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5536
5537	uint32_t u32ErrCode = 0;
5538	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5539	{
5540	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5541	AssertRCReturn(rc, rc);
5542	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5543	}
5544
5545	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5546	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5547	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5548	rc = VINF_SUCCESS;
5549	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5550	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5551
5552	break;
5553	}
5554
5555	case VMXREFLECTXCPT_DF:
5556	{
5557	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5558	rc = VINF_HM_DOUBLE_FAULT;
5559	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5560	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5561
5562	break;
5563	}
5564
5565	case VMXREFLECTXCPT_TF:
5566	{
5567	rc = VINF_EM_RESET;
5568	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5569	uExitVector));
5570	break;
5571	}
5572
5573	default:
5574	Assert(rc == VINF_SUCCESS);
5575	break;
5576	}
5577	}
5578	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5579	return rc;
5580	}
5581
5582
5583	/**
5584	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5585	*
5586	* @returns VBox status code.
5587	* @param pVCpu Pointer to the VMCPU.
5588	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5589	* out-of-sync. Make sure to update the required fields
5590	* before using them.
5591	*
5592	* @remarks No-long-jump zone!!!
5593	*/
5594	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5595	{
5596	NOREF(pMixedCtx);
5597
5598	/*
5599	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5600	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5601	*/
5602	VMMRZCallRing3Disable(pVCpu);
5603	HM_DISABLE_PREEMPT_IF_NEEDED();
5604
5605	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5606	{
5607	uint32_t uVal = 0;
5608	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5609	AssertRCReturn(rc, rc);
5610
5611	uint32_t uShadow = 0;
5612	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5613	AssertRCReturn(rc, rc);
5614
5615	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5616	CPUMSetGuestCR0(pVCpu, uVal);
5617	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5618	}
5619
5620	HM_RESTORE_PREEMPT_IF_NEEDED();
5621	VMMRZCallRing3Enable(pVCpu);
5622	return VINF_SUCCESS;
5623	}
5624
5625
5626	/**
5627	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5628	*
5629	* @returns VBox status code.
5630	* @param pVCpu Pointer to the VMCPU.
5631	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5632	* out-of-sync. Make sure to update the required fields
5633	* before using them.
5634	*
5635	* @remarks No-long-jump zone!!!
5636	*/
5637	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5638	{
5639	NOREF(pMixedCtx);
5640
5641	int rc = VINF_SUCCESS;
5642	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5643	{
5644	uint32_t uVal = 0;
5645	uint32_t uShadow = 0;
5646	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5647	AssertRCReturn(rc, rc);
5648	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5649	AssertRCReturn(rc, rc);
5650
5651	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5652	CPUMSetGuestCR4(pVCpu, uVal);
5653	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5654	}
5655	return rc;
5656	}
5657
5658
5659	/**
5660	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5661	*
5662	* @returns VBox status code.
5663	* @param pVCpu Pointer to the VMCPU.
5664	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5665	* out-of-sync. Make sure to update the required fields
5666	* before using them.
5667	*
5668	* @remarks No-long-jump zone!!!
5669	*/
5670	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5671	{
5672	int rc = VINF_SUCCESS;
5673	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5674	{
5675	uint64_t u64Val = 0;
5676	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5677	AssertRCReturn(rc, rc);
5678
5679	pMixedCtx->rip = u64Val;
5680	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5681	}
5682	return rc;
5683	}
5684
5685
5686	/**
5687	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5688	*
5689	* @returns VBox status code.
5690	* @param pVCpu Pointer to the VMCPU.
5691	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5692	* out-of-sync. Make sure to update the required fields
5693	* before using them.
5694	*
5695	* @remarks No-long-jump zone!!!
5696	*/
5697	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5698	{
5699	int rc = VINF_SUCCESS;
5700	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5701	{
5702	uint64_t u64Val = 0;
5703	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5704	AssertRCReturn(rc, rc);
5705
5706	pMixedCtx->rsp = u64Val;
5707	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5708	}
5709	return rc;
5710	}
5711
5712
5713	/**
5714	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5715	*
5716	* @returns VBox status code.
5717	* @param pVCpu Pointer to the VMCPU.
5718	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5719	* out-of-sync. Make sure to update the required fields
5720	* before using them.
5721	*
5722	* @remarks No-long-jump zone!!!
5723	*/
5724	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5725	{
5726	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5727	{
5728	uint32_t uVal = 0;
5729	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5730	AssertRCReturn(rc, rc);
5731
5732	pMixedCtx->eflags.u32 = uVal;
5733	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5734	{
5735	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5736	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5737
5738	pMixedCtx->eflags.Bits.u1VM = 0;
5739	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5740	}
5741
5742	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5743	}
5744	return VINF_SUCCESS;
5745	}
5746
5747
5748	/**
5749	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5750	* guest-CPU context.
5751	*/
5752	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753	{
5754	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5755	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5756	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5757	return rc;
5758	}
5759
5760
5761	/**
5762	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5763	* from the guest-state area in the VMCS.
5764	*
5765	* @param pVCpu Pointer to the VMCPU.
5766	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5767	* out-of-sync. Make sure to update the required fields
5768	* before using them.
5769	*
5770	* @remarks No-long-jump zone!!!
5771	*/
5772	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5773	{
5774	uint32_t uIntrState = 0;
5775	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5776	AssertRC(rc);
5777
5778	if (!uIntrState)
5779	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5780	else
5781	{
5782	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5783	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5784	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5785	AssertRC(rc);
5786	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5787	AssertRC(rc);
5788
5789	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5790	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5791	}
5792	}
5793
5794
5795	/**
5796	* Saves the guest's activity state.
5797	*
5798	* @returns VBox status code.
5799	* @param pVCpu Pointer to the VMCPU.
5800	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5801	* out-of-sync. Make sure to update the required fields
5802	* before using them.
5803	*
5804	* @remarks No-long-jump zone!!!
5805	*/
5806	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5807	{
5808	NOREF(pMixedCtx);
5809	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5810	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5811	return VINF_SUCCESS;
5812	}
5813
5814
5815	/**
5816	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5817	* the current VMCS into the guest-CPU context.
5818	*
5819	* @returns VBox status code.
5820	* @param pVCpu Pointer to the VMCPU.
5821	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5822	* out-of-sync. Make sure to update the required fields
5823	* before using them.
5824	*
5825	* @remarks No-long-jump zone!!!
5826	*/
5827	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5828	{
5829	int rc = VINF_SUCCESS;
5830	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5831	{
5832	uint32_t u32Val = 0;
5833	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5834	pMixedCtx->SysEnter.cs = u32Val;
5835	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5836	}
5837
5838	uint64_t u64Val = 0;
5839	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5840	{
5841	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5842	pMixedCtx->SysEnter.eip = u64Val;
5843	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5844	}
5845	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5846	{
5847	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5848	pMixedCtx->SysEnter.esp = u64Val;
5849	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5850	}
5851	return rc;
5852	}
5853
5854
5855	/**
5856	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5857	* the CPU back into the guest-CPU context.
5858	*
5859	* @returns VBox status code.
5860	* @param pVCpu Pointer to the VMCPU.
5861	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5862	* out-of-sync. Make sure to update the required fields
5863	* before using them.
5864	*
5865	* @remarks No-long-jump zone!!!
5866	*/
5867	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5868	{
5869	#if HC_ARCH_BITS == 64
5870	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5871	{
5872	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5873	VMMRZCallRing3Disable(pVCpu);
5874	HM_DISABLE_PREEMPT_IF_NEEDED();
5875
5876	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5877	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5878	{
5879	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5880	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5881	}
5882
5883	HM_RESTORE_PREEMPT_IF_NEEDED();
5884	VMMRZCallRing3Enable(pVCpu);
5885	}
5886	else
5887	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5888	#else
5889	NOREF(pMixedCtx);
5890	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5891	#endif
5892
5893	return VINF_SUCCESS;
5894	}
5895
5896
5897	/**
5898	* Saves the auto load/store'd guest MSRs from the current VMCS into
5899	* the guest-CPU context.
5900	*
5901	* @returns VBox status code.
5902	* @param pVCpu Pointer to the VMCPU.
5903	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5904	* out-of-sync. Make sure to update the required fields
5905	* before using them.
5906	*
5907	* @remarks No-long-jump zone!!!
5908	*/
5909	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5910	{
5911	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5912	return VINF_SUCCESS;
5913
5914	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5915	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5916	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5917	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5918	{
5919	switch (pMsr->u32Msr)
5920	{
5921	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5922	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5923	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5924	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5925	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5926	default:
5927	{
5928	AssertFailed();
5929	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5930	}
5931	}
5932	}
5933
5934	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5935	return VINF_SUCCESS;
5936	}
5937
5938
5939	/**
5940	* Saves the guest control registers from the current VMCS into the guest-CPU
5941	* context.
5942	*
5943	* @returns VBox status code.
5944	* @param pVCpu Pointer to the VMCPU.
5945	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5946	* out-of-sync. Make sure to update the required fields
5947	* before using them.
5948	*
5949	* @remarks No-long-jump zone!!!
5950	*/
5951	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5952	{
5953	/* Guest CR0. Guest FPU. */
5954	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5955	AssertRCReturn(rc, rc);
5956
5957	/* Guest CR4. */
5958	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5959	AssertRCReturn(rc, rc);
5960
5961	/* Guest CR2 - updated always during the world-switch or in #PF. */
5962	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5963	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5964	{
5965	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5966	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5967
5968	PVM pVM = pVCpu->CTX_SUFF(pVM);
5969	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5970	\|\| ( pVM->hm.s.fNestedPaging
5971	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5972	{
5973	uint64_t u64Val = 0;
5974	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5975	if (pMixedCtx->cr3 != u64Val)
5976	{
5977	CPUMSetGuestCR3(pVCpu, u64Val);
5978	if (VMMRZCallRing3IsEnabled(pVCpu))
5979	{
5980	PGMUpdateCR3(pVCpu, u64Val);
5981	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5982	}
5983	else
5984	{
5985	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5986	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5987	}
5988	}
5989
5990	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5991	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5992	{
5993	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5994	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5995	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5996	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5997
5998	if (VMMRZCallRing3IsEnabled(pVCpu))
5999	{
6000	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6001	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6002	}
6003	else
6004	{
6005	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6006	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6007	}
6008	}
6009	}
6010
6011	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6012	}
6013
6014	/*
6015	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6016	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6017	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6018	*
6019	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6020	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6021	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6022	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6023	*
6024	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6025	*/
6026	if (VMMRZCallRing3IsEnabled(pVCpu))
6027	{
6028	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6029	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6030
6031	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6032	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6033
6034	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6035	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6036	}
6037
6038	return rc;
6039	}
6040
6041
6042	/**
6043	* Reads a guest segment register from the current VMCS into the guest-CPU
6044	* context.
6045	*
6046	* @returns VBox status code.
6047	* @param pVCpu Pointer to the VMCPU.
6048	* @param idxSel Index of the selector in the VMCS.
6049	* @param idxLimit Index of the segment limit in the VMCS.
6050	* @param idxBase Index of the segment base in the VMCS.
6051	* @param idxAccess Index of the access rights of the segment in the VMCS.
6052	* @param pSelReg Pointer to the segment selector.
6053	*
6054	* @remarks No-long-jump zone!!!
6055	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6056	* macro as that takes care of whether to read from the VMCS cache or
6057	* not.
6058	*/
6059	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6060	PCPUMSELREG pSelReg)
6061	{
6062	NOREF(pVCpu);
6063
6064	uint32_t u32Val = 0;
6065	int rc = VMXReadVmcs32(idxSel, &u32Val);
6066	AssertRCReturn(rc, rc);
6067	pSelReg->Sel = (uint16_t)u32Val;
6068	pSelReg->ValidSel = (uint16_t)u32Val;
6069	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6070
6071	rc = VMXReadVmcs32(idxLimit, &u32Val);
6072	AssertRCReturn(rc, rc);
6073	pSelReg->u32Limit = u32Val;
6074
6075	uint64_t u64Val = 0;
6076	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6077	AssertRCReturn(rc, rc);
6078	pSelReg->u64Base = u64Val;
6079
6080	rc = VMXReadVmcs32(idxAccess, &u32Val);
6081	AssertRCReturn(rc, rc);
6082	pSelReg->Attr.u = u32Val;
6083
6084	/*
6085	* If VT-x marks the segment as unusable, most other bits remain undefined:
6086	* - For CS the L, D and G bits have meaning.
6087	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6088	* - For the remaining data segments no bits are defined.
6089	*
6090	* The present bit and the unusable bit has been observed to be set at the
6091	* same time (the selector was supposed to invalid as we started executing
6092	* a V8086 interrupt in ring-0).
6093	*
6094	* What should be important for the rest of the VBox code, is that the P bit is
6095	* cleared. Some of the other VBox code recognizes the unusable bit, but
6096	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6097	* safe side here, we'll strip off P and other bits we don't care about. If
6098	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6099	*
6100	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6101	*/
6102	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6103	{
6104	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6105
6106	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6107	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6108	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6109
6110	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6111	#ifdef DEBUG_bird
6112	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6113	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6114	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6115	#endif
6116	}
6117	return VINF_SUCCESS;
6118	}
6119
6120
6121	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6122	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6123	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6124	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6125	#else
6126	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6127	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6128	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6129	#endif
6130
6131
6132	/**
6133	* Saves the guest segment registers from the current VMCS into the guest-CPU
6134	* context.
6135	*
6136	* @returns VBox status code.
6137	* @param pVCpu Pointer to the VMCPU.
6138	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6139	* out-of-sync. Make sure to update the required fields
6140	* before using them.
6141	*
6142	* @remarks No-long-jump zone!!!
6143	*/
6144	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6145	{
6146	/* Guest segment registers. */
6147	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6148	{
6149	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6150	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6151	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6152	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6153	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6154	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6155	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6156
6157	/* Restore segment attributes for real-on-v86 mode hack. */
6158	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6159	{
6160	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6161	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6162	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6163	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6164	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6165	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6166	}
6167	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6168	}
6169
6170	return VINF_SUCCESS;
6171	}
6172
6173
6174	/**
6175	* Saves the guest descriptor table registers and task register from the current
6176	* VMCS into the guest-CPU context.
6177	*
6178	* @returns VBox status code.
6179	* @param pVCpu Pointer to the VMCPU.
6180	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6181	* out-of-sync. Make sure to update the required fields
6182	* before using them.
6183	*
6184	* @remarks No-long-jump zone!!!
6185	*/
6186	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6187	{
6188	int rc = VINF_SUCCESS;
6189
6190	/* Guest LDTR. */
6191	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6192	{
6193	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6194	AssertRCReturn(rc, rc);
6195	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6196	}
6197
6198	/* Guest GDTR. */
6199	uint64_t u64Val = 0;
6200	uint32_t u32Val = 0;
6201	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6202	{
6203	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6204	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6205	pMixedCtx->gdtr.pGdt = u64Val;
6206	pMixedCtx->gdtr.cbGdt = u32Val;
6207	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6208	}
6209
6210	/* Guest IDTR. */
6211	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6212	{
6213	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6214	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6215	pMixedCtx->idtr.pIdt = u64Val;
6216	pMixedCtx->idtr.cbIdt = u32Val;
6217	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6218	}
6219
6220	/* Guest TR. */
6221	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6222	{
6223	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6224	AssertRCReturn(rc, rc);
6225
6226	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6227	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6228	{
6229	rc = VMXLOCAL_READ_SEG(TR, tr);
6230	AssertRCReturn(rc, rc);
6231	}
6232	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6233	}
6234	return rc;
6235	}
6236
6237	#undef VMXLOCAL_READ_SEG
6238
6239
6240	/**
6241	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6242	* context.
6243	*
6244	* @returns VBox status code.
6245	* @param pVCpu Pointer to the VMCPU.
6246	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6247	* out-of-sync. Make sure to update the required fields
6248	* before using them.
6249	*
6250	* @remarks No-long-jump zone!!!
6251	*/
6252	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6253	{
6254	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6255	{
6256	if (!pVCpu->hm.s.fUsingHyperDR7)
6257	{
6258	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6259	uint32_t u32Val;
6260	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6261	pMixedCtx->dr[7] = u32Val;
6262	}
6263
6264	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6265	}
6266	return VINF_SUCCESS;
6267	}
6268
6269
6270	/**
6271	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6272	*
6273	* @returns VBox status code.
6274	* @param pVCpu Pointer to the VMCPU.
6275	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6276	* out-of-sync. Make sure to update the required fields
6277	* before using them.
6278	*
6279	* @remarks No-long-jump zone!!!
6280	*/
6281	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282	{
6283	NOREF(pMixedCtx);
6284
6285	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6286	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6287	return VINF_SUCCESS;
6288	}
6289
6290
6291	/**
6292	* Saves the entire guest state from the currently active VMCS into the
6293	* guest-CPU context. This essentially VMREADs all guest-data.
6294	*
6295	* @returns VBox status code.
6296	* @param pVCpu Pointer to the VMCPU.
6297	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6298	* out-of-sync. Make sure to update the required fields
6299	* before using them.
6300	*/
6301	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6302	{
6303	Assert(pVCpu);
6304	Assert(pMixedCtx);
6305
6306	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6307	return VINF_SUCCESS;
6308
6309	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6310	again on the ring-3 callback path, there is no real need to. */
6311	if (VMMRZCallRing3IsEnabled(pVCpu))
6312	VMMR0LogFlushDisable(pVCpu);
6313	else
6314	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6315	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6316
6317	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6318	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6319
6320	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6321	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6322
6323	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6324	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6325
6326	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6327	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6328
6329	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6330	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6331
6332	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6333	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6334
6335	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6336	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6337
6338	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6339	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6340
6341	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6342	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6343
6344	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6345	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6346
6347	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6348	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6349
6350	if (VMMRZCallRing3IsEnabled(pVCpu))
6351	VMMR0LogFlushEnable(pVCpu);
6352
6353	return rc;
6354	}
6355
6356
6357	/**
6358	* Check per-VM and per-VCPU force flag actions that require us to go back to
6359	* ring-3 for one reason or another.
6360	*
6361	* @returns VBox status code (information status code included).
6362	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6363	* ring-3.
6364	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6365	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6366	* interrupts)
6367	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6368	* all EMTs to be in ring-3.
6369	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6370	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6371	* to the EM loop.
6372	*
6373	* @param pVM Pointer to the VM.
6374	* @param pVCpu Pointer to the VMCPU.
6375	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6376	* out-of-sync. Make sure to update the required fields
6377	* before using them.
6378	*/
6379	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6380	{
6381	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6382
6383	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6384	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6385	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6386	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6387	{
6388	/* We need the control registers now, make sure the guest-CPU context is updated. */
6389	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6390	AssertRCReturn(rc3, rc3);
6391
6392	/* Pending HM CR3 sync. */
6393	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6394	{
6395	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6396	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6397	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6398	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6399	}
6400
6401	/* Pending HM PAE PDPEs. */
6402	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6403	{
6404	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6405	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6406	}
6407
6408	/* Pending PGM C3 sync. */
6409	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6410	{
6411	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6412	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6413	if (rc2 != VINF_SUCCESS)
6414	{
6415	AssertRC(rc2);
6416	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6417	return rc2;
6418	}
6419	}
6420
6421	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6422	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6423	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6424	{
6425	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6426	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6427	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6428	return rc2;
6429	}
6430
6431	/* Pending VM request packets, such as hardware interrupts. */
6432	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6433	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6434	{
6435	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6436	return VINF_EM_PENDING_REQUEST;
6437	}
6438
6439	/* Pending PGM pool flushes. */
6440	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6441	{
6442	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6443	return VINF_PGM_POOL_FLUSH_PENDING;
6444	}
6445
6446	/* Pending DMA requests. */
6447	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6448	{
6449	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6450	return VINF_EM_RAW_TO_R3;
6451	}
6452	}
6453
6454	return VINF_SUCCESS;
6455	}
6456
6457
6458	/**
6459	* Converts any TRPM trap into a pending HM event. This is typically used when
6460	* entering from ring-3 (not longjmp returns).
6461	*
6462	* @param pVCpu Pointer to the VMCPU.
6463	*/
6464	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6465	{
6466	Assert(TRPMHasTrap(pVCpu));
6467	Assert(!pVCpu->hm.s.Event.fPending);
6468
6469	uint8_t uVector;
6470	TRPMEVENT enmTrpmEvent;
6471	RTGCUINT uErrCode;
6472	RTGCUINTPTR GCPtrFaultAddress;
6473	uint8_t cbInstr;
6474
6475	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6476	AssertRC(rc);
6477
6478	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6479	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6480	if (enmTrpmEvent == TRPM_TRAP)
6481	{
6482	switch (uVector)
6483	{
6484	case X86_XCPT_BP:
6485	case X86_XCPT_OF:
6486	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6487	break;
6488
6489	case X86_XCPT_PF:
6490	case X86_XCPT_DF:
6491	case X86_XCPT_TS:
6492	case X86_XCPT_NP:
6493	case X86_XCPT_SS:
6494	case X86_XCPT_GP:
6495	case X86_XCPT_AC:
6496	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6497	/* no break! */
6498	default:
6499	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6500	break;
6501	}
6502	}
6503	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6504	{
6505	if (uVector == X86_XCPT_NMI)
6506	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6507	else
6508	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6509	}
6510	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6511	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6512	else
6513	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6514
6515	rc = TRPMResetTrap(pVCpu);
6516	AssertRC(rc);
6517	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6518	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6519
6520	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6521	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6522	}
6523
6524
6525	/**
6526	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6527	* VT-x to execute any instruction.
6528	*
6529	* @param pvCpu Pointer to the VMCPU.
6530	*/
6531	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6532	{
6533	Assert(pVCpu->hm.s.Event.fPending);
6534
6535	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6536	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6537	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6538	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6539
6540	/* If a trap was already pending, we did something wrong! */
6541	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6542
6543	TRPMEVENT enmTrapType;
6544	switch (uVectorType)
6545	{
6546	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6547	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6548	enmTrapType = TRPM_HARDWARE_INT;
6549	break;
6550
6551	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6552	enmTrapType = TRPM_SOFTWARE_INT;
6553	break;
6554
6555	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6556	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6557	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6558	enmTrapType = TRPM_TRAP;
6559	break;
6560
6561	default:
6562	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6563	enmTrapType = TRPM_32BIT_HACK;
6564	break;
6565	}
6566
6567	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6568
6569	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6570	AssertRC(rc);
6571
6572	if (fErrorCodeValid)
6573	TRPMSetErrorCode(pVCpu, uErrorCode);
6574
6575	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6576	&& uVector == X86_XCPT_PF)
6577	{
6578	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6579	}
6580	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6581	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6582	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6583	{
6584	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6585	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6586	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6587	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6588	}
6589	pVCpu->hm.s.Event.fPending = false;
6590	}
6591
6592
6593	/**
6594	* Does the necessary state syncing before returning to ring-3 for any reason
6595	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6596	*
6597	* @returns VBox status code.
6598	* @param pVM Pointer to the VM.
6599	* @param pVCpu Pointer to the VMCPU.
6600	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6601	* be out-of-sync. Make sure to update the required
6602	* fields before using them.
6603	* @param fSaveGuestState Whether to save the guest state or not.
6604	*
6605	* @remarks No-long-jmp zone!!!
6606	*/
6607	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6608	{
6609	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6610	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6611
6612	RTCPUID idCpu = RTMpCpuId();
6613	Log4Func(("HostCpuId=%u\n", idCpu));
6614
6615	/*
6616	* !!! IMPORTANT !!!
6617	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6618	*/
6619
6620	/* Save the guest state if necessary. */
6621	if ( fSaveGuestState
6622	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6623	{
6624	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6625	AssertRCReturn(rc, rc);
6626	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6627	}
6628
6629	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6630	if (CPUMIsGuestFPUStateActive(pVCpu))
6631	{
6632	/* We shouldn't reload CR0 without saving it first. */
6633	if (!fSaveGuestState)
6634	{
6635	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6636	AssertRCReturn(rc, rc);
6637	}
6638	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6639	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6640	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6641	}
6642
6643	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6644	#ifdef VBOX_STRICT
6645	if (CPUMIsHyperDebugStateActive(pVCpu))
6646	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6647	#endif
6648	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6649	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6650	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6651	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6652
6653	#if HC_ARCH_BITS == 64
6654	/* Restore host-state bits that VT-x only restores partially. */
6655	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6656	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6657	{
6658	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6659	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6660	}
6661	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6662	#endif
6663
6664	#if HC_ARCH_BITS == 64
6665	/* Restore the host MSRs as we're leaving VT-x context. */
6666	if ( pVM->hm.s.fAllow64BitGuests
6667	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6668	{
6669	/* We shouldn't reload the guest MSRs without saving it first. */
6670	if (!fSaveGuestState)
6671	{
6672	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6673	AssertRCReturn(rc, rc);
6674	}
6675	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6676	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6677	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6678	}
6679	#endif
6680
6681	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6682	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6683
6684	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6685	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6686	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6687	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6688	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6689	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6690	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6691	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6692
6693	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6694
6695	/** @todo This partially defeats the purpose of having preemption hooks.
6696	* The problem is, deregistering the hooks should be moved to a place that
6697	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6698	* context.
6699	*/
6700	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6701	{
6702	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6703	AssertRCReturn(rc, rc);
6704
6705	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6706	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6707	}
6708	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6709	NOREF(idCpu);
6710
6711	return VINF_SUCCESS;
6712	}
6713
6714
6715	/**
6716	* Leaves the VT-x session.
6717	*
6718	* @returns VBox status code.
6719	* @param pVM Pointer to the VM.
6720	* @param pVCpu Pointer to the VMCPU.
6721	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6722	* out-of-sync. Make sure to update the required fields
6723	* before using them.
6724	*
6725	* @remarks No-long-jmp zone!!!
6726	*/
6727	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6728	{
6729	HM_DISABLE_PREEMPT_IF_NEEDED();
6730	HMVMX_ASSERT_CPU_SAFE();
6731	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6732	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6733
6734	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6735	and done this from the VMXR0ThreadCtxCallback(). */
6736	if (!pVCpu->hm.s.fLeaveDone)
6737	{
6738	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6739	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6740	pVCpu->hm.s.fLeaveDone = true;
6741	}
6742
6743	/*
6744	* !!! IMPORTANT !!!
6745	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6746	*/
6747
6748	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6749	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6750	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6751	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6752	VMMR0ThreadCtxHooksDeregister(pVCpu);
6753
6754	/* Leave HM context. This takes care of local init (term). */
6755	int rc = HMR0LeaveCpu(pVCpu);
6756
6757	HM_RESTORE_PREEMPT_IF_NEEDED();
6758
6759	return rc;
6760	}
6761
6762
6763	/**
6764	* Does the necessary state syncing before doing a longjmp to ring-3.
6765	*
6766	* @returns VBox status code.
6767	* @param pVM Pointer to the VM.
6768	* @param pVCpu Pointer to the VMCPU.
6769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6770	* out-of-sync. Make sure to update the required fields
6771	* before using them.
6772	*
6773	* @remarks No-long-jmp zone!!!
6774	*/
6775	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6776	{
6777	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6778	}
6779
6780
6781	/**
6782	* Take necessary actions before going back to ring-3.
6783	*
6784	* An action requires us to go back to ring-3. This function does the necessary
6785	* steps before we can safely return to ring-3. This is not the same as longjmps
6786	* to ring-3, this is voluntary and prepares the guest so it may continue
6787	* executing outside HM (recompiler/IEM).
6788	*
6789	* @returns VBox status code.
6790	* @param pVM Pointer to the VM.
6791	* @param pVCpu Pointer to the VMCPU.
6792	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6793	* out-of-sync. Make sure to update the required fields
6794	* before using them.
6795	* @param rcExit The reason for exiting to ring-3. Can be
6796	* VINF_VMM_UNKNOWN_RING3_CALL.
6797	*/
6798	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6799	{
6800	Assert(pVM);
6801	Assert(pVCpu);
6802	Assert(pMixedCtx);
6803	HMVMX_ASSERT_PREEMPT_SAFE();
6804
6805	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6806	{
6807	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6808	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6809	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6810	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6811	}
6812
6813	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6814	VMMRZCallRing3Disable(pVCpu);
6815	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6816
6817	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6818	if (pVCpu->hm.s.Event.fPending)
6819	{
6820	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6821	Assert(!pVCpu->hm.s.Event.fPending);
6822	}
6823
6824	/* Save guest state and restore host state bits. */
6825	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6826	AssertRCReturn(rc, rc);
6827	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6828
6829	/* Sync recompiler state. */
6830	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6831	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6832	\| CPUM_CHANGED_LDTR
6833	\| CPUM_CHANGED_GDTR
6834	\| CPUM_CHANGED_IDTR
6835	\| CPUM_CHANGED_TR
6836	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6837	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6838	if ( pVM->hm.s.fNestedPaging
6839	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6840	{
6841	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6842	}
6843
6844	Assert(!pVCpu->hm.s.fClearTrapFlag);
6845
6846	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6847	if (rcExit != VINF_EM_RAW_INTERRUPT)
6848	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6849
6850	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6851
6852	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6853	VMMRZCallRing3RemoveNotification(pVCpu);
6854	VMMRZCallRing3Enable(pVCpu);
6855
6856	return rc;
6857	}
6858
6859
6860	/**
6861	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6862	* longjump to ring-3 and possibly get preempted.
6863	*
6864	* @returns VBox status code.
6865	* @param pVCpu Pointer to the VMCPU.
6866	* @param enmOperation The operation causing the ring-3 longjump.
6867	* @param pvUser Opaque pointer to the guest-CPU context. The data
6868	* may be out-of-sync. Make sure to update the required
6869	* fields before using them.
6870	*/
6871	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6872	{
6873	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6874	{
6875	/*
6876	* !!! IMPORTANT !!!
6877	* If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6878	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6879	*/
6880	VMMRZCallRing3RemoveNotification(pVCpu);
6881	VMMRZCallRing3Disable(pVCpu);
6882	HM_DISABLE_PREEMPT_IF_NEEDED();
6883
6884	PVM pVM = pVCpu->CTX_SUFF(pVM);
6885	if (CPUMIsGuestFPUStateActive(pVCpu))
6886	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6887
6888	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6889
6890	#if HC_ARCH_BITS == 64
6891	/* Restore host-state bits that VT-x only restores partially. */
6892	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6893	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6894	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6895	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6896
6897	/* Restore the host MSRs as we're leaving VT-x context. */
6898	if ( pVM->hm.s.fAllow64BitGuests
6899	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6900	{
6901	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6902	}
6903	#endif
6904	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6905	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6906	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6907	{
6908	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6909	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6910	}
6911
6912	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6913	VMMR0ThreadCtxHooksDeregister(pVCpu);
6914
6915	HMR0LeaveCpu(pVCpu);
6916	HM_RESTORE_PREEMPT_IF_NEEDED();
6917	return VINF_SUCCESS;
6918	}
6919
6920	Assert(pVCpu);
6921	Assert(pvUser);
6922	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6923	HMVMX_ASSERT_PREEMPT_SAFE();
6924
6925	VMMRZCallRing3Disable(pVCpu);
6926	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6927
6928	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6929	enmOperation));
6930
6931	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6932	AssertRCReturn(rc, rc);
6933
6934	VMMRZCallRing3Enable(pVCpu);
6935	return VINF_SUCCESS;
6936	}
6937
6938
6939	/**
6940	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6941	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6942	*
6943	* @param pVCpu Pointer to the VMCPU.
6944	*/
6945	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6946	{
6947	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6948	{
6949	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6950	{
6951	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6953	AssertRC(rc);
6954	Log4(("Setup interrupt-window exiting\n"));
6955	}
6956	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6957	}
6958
6959
6960	/**
6961	* Clears the interrupt-window exiting control in the VMCS.
6962	*
6963	* @param pVCpu Pointer to the VMCPU.
6964	*/
6965	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
6966	{
6967	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
6968	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6969	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6970	AssertRC(rc);
6971	Log4(("Cleared interrupt-window exiting\n"));
6972	}
6973
6974
6975	/**
6976	* Evaluates the event to be delivered to the guest and sets it as the pending
6977	* event.
6978	*
6979	* @param pVCpu Pointer to the VMCPU.
6980	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6981	* out-of-sync. Make sure to update the required fields
6982	* before using them.
6983	*/
6984	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6985	{
6986	Assert(!pVCpu->hm.s.Event.fPending);
6987
6988	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6989	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6990	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6991	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6992
6993	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
6994	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6995	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6996	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6997	Assert(!TRPMHasTrap(pVCpu));
6998
6999	/** @todo SMI. SMIs take priority over NMIs. */
7000	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7001	{
7002	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7003	if ( !fBlockMovSS
7004	&& !fBlockSti)
7005	{
7006	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7007	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7008	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7009	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7010
7011	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
7012	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7013	}
7014	else
7015	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7016	}
7017	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7018	&& !pVCpu->hm.s.fSingleInstruction)
7019	{
7020	/*
7021	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7022	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
7023	* evaluated here and not set as pending, solely based on the force-flags.
7024	*/
7025	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7026	AssertRC(rc);
7027	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7028	if ( !fBlockInt
7029	&& !fBlockSti
7030	&& !fBlockMovSS)
7031	{
7032	uint8_t u8Interrupt;
7033	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7034	if (RT_SUCCESS(rc))
7035	{
7036	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7037	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7038	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7039
7040	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7041	}
7042	else
7043	{
7044	/** @todo Does this actually happen? If not turn it into an assertion. */
7045	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7046	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7047	}
7048	}
7049	else
7050	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7051	}
7052	}
7053
7054
7055	/**
7056	* Sets a pending-debug exception to be delivered to the guest if the guest is
7057	* single-stepping.
7058	*
7059	* @param pVCpu Pointer to the VMCPU.
7060	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7061	* out-of-sync. Make sure to update the required fields
7062	* before using them.
7063	*/
7064	DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7065	{
7066	HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
7067	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7068	{
7069	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7070	AssertRC(rc);
7071	}
7072	}
7073
7074
7075	/**
7076	* Injects any pending events into the guest if the guest is in a state to
7077	* receive them.
7078	*
7079	* @returns VBox status code (informational status codes included).
7080	* @param pVCpu Pointer to the VMCPU.
7081	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7082	* out-of-sync. Make sure to update the required fields
7083	* before using them.
7084	*/
7085	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7086	{
7087	HMVMX_ASSERT_PREEMPT_SAFE();
7088	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7089
7090	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7091	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7092	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7093	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7094
7095	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7096	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7097	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7098	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7099	Assert(!TRPMHasTrap(pVCpu));
7100
7101	int rc = VINF_SUCCESS;
7102	if (pVCpu->hm.s.Event.fPending)
7103	{
7104	/*
7105	* Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7106	* VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7107	* ended up enabling interrupts outside VT-x.
7108	*/
7109	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7110	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7111	&& ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7112	\|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7113	{
7114	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7115	hmR0VmxClearIntWindowExitVmcs(pVCpu);
7116	}
7117	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7118	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7119	{
7120	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7121	AssertRCReturn(rc, rc);
7122	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7123	Assert(!fBlockInt);
7124	Assert(!fBlockSti);
7125	Assert(!fBlockMovSS);
7126	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7127	}
7128	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7129	{
7130	Assert(!fBlockSti);
7131	Assert(!fBlockMovSS);
7132	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7133	}
7134	#endif
7135	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7136	(uint8_t)uIntType));
7137	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7138	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7139	AssertRCReturn(rc, rc);
7140
7141	/* Update the interruptibility-state as it could have been changed by
7142	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7143	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7144	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7145
7146	#ifdef VBOX_WITH_STATISTICS
7147	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7148	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7149	else
7150	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7151	#endif
7152	}
7153
7154	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7155	if ( fBlockSti
7156	\|\| fBlockMovSS)
7157	{
7158	if ( !pVCpu->hm.s.fSingleInstruction
7159	&& !DBGFIsStepping(pVCpu))
7160	{
7161	/*
7162	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7163	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7164	* See Intel spec. 27.3.4 "Saving Non-Register State".
7165	*/
7166	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7167	AssertRCReturn(rc2, rc2);
7168	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7169	}
7170	else if (pMixedCtx->eflags.Bits.u1TF)
7171	{
7172	/*
7173	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7174	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7175	*/
7176	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7177	uIntrState = 0;
7178	}
7179	}
7180
7181	/*
7182	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7183	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7184	*/
7185	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7186	AssertRC(rc2);
7187
7188	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7189	NOREF(fBlockMovSS); NOREF(fBlockSti);
7190	return rc;
7191	}
7192
7193
7194	/**
7195	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7196	*
7197	* @param pVCpu Pointer to the VMCPU.
7198	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7199	* out-of-sync. Make sure to update the required fields
7200	* before using them.
7201	*/
7202	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7203	{
7204	NOREF(pMixedCtx);
7205	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7206	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7207	}
7208
7209
7210	/**
7211	* Injects a double-fault (#DF) exception into the VM.
7212	*
7213	* @returns VBox status code (informational status code included).
7214	* @param pVCpu Pointer to the VMCPU.
7215	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7216	* out-of-sync. Make sure to update the required fields
7217	* before using them.
7218	*/
7219	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7220	{
7221	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7222	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7223	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7224	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7225	puIntrState);
7226	}
7227
7228
7229	/**
7230	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7231	*
7232	* @param pVCpu Pointer to the VMCPU.
7233	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7234	* out-of-sync. Make sure to update the required fields
7235	* before using them.
7236	*/
7237	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7238	{
7239	NOREF(pMixedCtx);
7240	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7241	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7242	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7243	}
7244
7245
7246	/**
7247	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7248	*
7249	* @param pVCpu Pointer to the VMCPU.
7250	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7251	* out-of-sync. Make sure to update the required fields
7252	* before using them.
7253	* @param cbInstr The value of RIP that is to be pushed on the guest
7254	* stack.
7255	*/
7256	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7257	{
7258	NOREF(pMixedCtx);
7259	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7260	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7261	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7262	}
7263
7264
7265	/**
7266	* Injects a general-protection (#GP) fault into the VM.
7267	*
7268	* @returns VBox status code (informational status code included).
7269	* @param pVCpu Pointer to the VMCPU.
7270	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7271	* out-of-sync. Make sure to update the required fields
7272	* before using them.
7273	* @param u32ErrorCode The error code associated with the #GP.
7274	*/
7275	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7276	uint32_t *puIntrState)
7277	{
7278	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7279	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7280	if (fErrorCodeValid)
7281	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7282	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7283	puIntrState);
7284	}
7285
7286
7287	/**
7288	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7289	*
7290	* @param pVCpu Pointer to the VMCPU.
7291	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7292	* out-of-sync. Make sure to update the required fields
7293	* before using them.
7294	* @param uVector The software interrupt vector number.
7295	* @param cbInstr The value of RIP that is to be pushed on the guest
7296	* stack.
7297	*/
7298	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7299	{
7300	NOREF(pMixedCtx);
7301	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7302	if ( uVector == X86_XCPT_BP
7303	\|\| uVector == X86_XCPT_OF)
7304	{
7305	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7306	}
7307	else
7308	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7309	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7310	}
7311
7312
7313	/**
7314	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7315	* stack.
7316	*
7317	* @returns VBox status code (information status code included).
7318	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7319	* @param pVM Pointer to the VM.
7320	* @param pMixedCtx Pointer to the guest-CPU context.
7321	* @param uValue The value to push to the guest stack.
7322	*/
7323	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7324	{
7325	/*
7326	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7327	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7328	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7329	*/
7330	if (pMixedCtx->sp == 1)
7331	return VINF_EM_RESET;
7332	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7333	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7334	AssertRCReturn(rc, rc);
7335	return rc;
7336	}
7337
7338
7339	/**
7340	* Injects an event into the guest upon VM-entry by updating the relevant fields
7341	* in the VM-entry area in the VMCS.
7342	*
7343	* @returns VBox status code (informational error codes included).
7344	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7345	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7346	*
7347	* @param pVCpu Pointer to the VMCPU.
7348	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7349	* be out-of-sync. Make sure to update the required
7350	* fields before using them.
7351	* @param u64IntInfo The VM-entry interruption-information field.
7352	* @param cbInstr The VM-entry instruction length in bytes (for
7353	* software interrupts, exceptions and privileged
7354	* software exceptions).
7355	* @param u32ErrCode The VM-entry exception error code.
7356	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7357	* @param puIntrState Pointer to the current guest interruptibility-state.
7358	* This interruptibility-state will be updated if
7359	* necessary. This cannot not be NULL.
7360	*
7361	* @remarks Requires CR0!
7362	* @remarks No-long-jump zone!!!
7363	*/
7364	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7365	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7366	{
7367	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7368	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7369	Assert(puIntrState);
7370	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7371
7372	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7373	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7374
7375	#ifdef VBOX_STRICT
7376	/* Validate the error-code-valid bit for hardware exceptions. */
7377	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7378	{
7379	switch (uVector)
7380	{
7381	case X86_XCPT_PF:
7382	case X86_XCPT_DF:
7383	case X86_XCPT_TS:
7384	case X86_XCPT_NP:
7385	case X86_XCPT_SS:
7386	case X86_XCPT_GP:
7387	case X86_XCPT_AC:
7388	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7389	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7390	/* fallthru */
7391	default:
7392	break;
7393	}
7394	}
7395	#endif
7396
7397	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7398	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7399	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7400
7401	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7402
7403	/* We require CR0 to check if the guest is in real-mode. */
7404	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7405	AssertRCReturn(rc, rc);
7406
7407	/*
7408	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7409	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7410	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7411	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7412	*/
7413	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7414	{
7415	PVM pVM = pVCpu->CTX_SUFF(pVM);
7416	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7417	{
7418	Assert(PDMVmmDevHeapIsEnabled(pVM));
7419	Assert(pVM->hm.s.vmx.pRealModeTSS);
7420
7421	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7422	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7423	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7424	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7425	AssertRCReturn(rc, rc);
7426	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7427
7428	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7429	const size_t cbIdtEntry = sizeof(X86IDTR16);
7430	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7431	{
7432	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7433	if (uVector == X86_XCPT_DF)
7434	return VINF_EM_RESET;
7435	else if (uVector == X86_XCPT_GP)
7436	{
7437	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7438	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7439	}
7440
7441	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7442	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7443	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7444	}
7445
7446	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7447	uint16_t uGuestIp = pMixedCtx->ip;
7448	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7449	{
7450	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7451	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7452	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7453	}
7454	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7455	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7456
7457	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7458	X86IDTR16 IdtEntry;
7459	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7460	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7461	AssertRCReturn(rc, rc);
7462
7463	/* Construct the stack frame for the interrupt/exception handler. */
7464	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7465	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7466	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7467	AssertRCReturn(rc, rc);
7468
7469	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7470	if (rc == VINF_SUCCESS)
7471	{
7472	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7473	pMixedCtx->rip = IdtEntry.offSel;
7474	pMixedCtx->cs.Sel = IdtEntry.uSel;
7475	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7476	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7477	&& uVector == X86_XCPT_PF)
7478	{
7479	pMixedCtx->cr2 = GCPtrFaultAddress;
7480	}
7481
7482	/* If any other guest-state bits are changed here, make sure to update
7483	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7484	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7485	\| HM_CHANGED_GUEST_RIP
7486	\| HM_CHANGED_GUEST_RFLAGS
7487	\| HM_CHANGED_GUEST_RSP);
7488
7489	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7490	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7491	{
7492	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7493	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7494	Log4(("Clearing inhibition due to STI.\n"));
7495	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7496	}
7497	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7498
7499	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7500	it, if we are returning to ring-3 before executing guest code. */
7501	pVCpu->hm.s.Event.fPending = false;
7502	}
7503	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7504	return rc;
7505	}
7506	else
7507	{
7508	/*
7509	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7510	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7511	*/
7512	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7513	}
7514	}
7515
7516	/* Validate. */
7517	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7518	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7519	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7520
7521	/* Inject. */
7522	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7523	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7524	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7525	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7526
7527	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7528	&& uVector == X86_XCPT_PF)
7529	{
7530	pMixedCtx->cr2 = GCPtrFaultAddress;
7531	}
7532
7533	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7534	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7535
7536	AssertRCReturn(rc, rc);
7537	return rc;
7538	}
7539
7540
7541	/**
7542	* Clears the interrupt-window exiting control in the VMCS and if necessary
7543	* clears the current event in the VMCS as well.
7544	*
7545	* @returns VBox status code.
7546	* @param pVCpu Pointer to the VMCPU.
7547	*
7548	* @remarks Use this function only to clear events that have not yet been
7549	* delivered to the guest but are injected in the VMCS!
7550	* @remarks No-long-jump zone!!!
7551	*/
7552	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7553	{
7554	int rc;
7555	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7556
7557	/* Clear interrupt-window exiting control. */
7558	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7559	hmR0VmxClearIntWindowExitVmcs(pVCpu);
7560
7561	if (!pVCpu->hm.s.Event.fPending)
7562	return;
7563
7564	#ifdef VBOX_STRICT
7565	uint32_t u32EntryInfo;
7566	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7567	AssertRC(rc);
7568	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7569	#endif
7570
7571	/* Clear the entry-interruption field (including the valid bit). */
7572	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7573	AssertRC(rc);
7574
7575	/* Clear the pending debug exception field. */
7576	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7577	AssertRC(rc);
7578	}
7579
7580
7581	/**
7582	* Enters the VT-x session.
7583	*
7584	* @returns VBox status code.
7585	* @param pVM Pointer to the VM.
7586	* @param pVCpu Pointer to the VMCPU.
7587	* @param pCpu Pointer to the CPU info struct.
7588	*/
7589	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7590	{
7591	AssertPtr(pVM);
7592	AssertPtr(pVCpu);
7593	Assert(pVM->hm.s.vmx.fSupported);
7594	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7595	NOREF(pCpu); NOREF(pVM);
7596
7597	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7598	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7599
7600	#ifdef VBOX_STRICT
7601	/* Make sure we're in VMX root mode. */
7602	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7603	if (!(u32HostCR4 & X86_CR4_VMXE))
7604	{
7605	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7606	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7607	}
7608	#endif
7609
7610	/*
7611	* Load the VCPU's VMCS as the current (and active) one.
7612	*/
7613	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7614	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7615	if (RT_FAILURE(rc))
7616	return rc;
7617
7618	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7619	pVCpu->hm.s.fLeaveDone = false;
7620	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7621
7622	return VINF_SUCCESS;
7623	}
7624
7625
7626	/**
7627	* The thread-context callback (only on platforms which support it).
7628	*
7629	* @param enmEvent The thread-context event.
7630	* @param pVCpu Pointer to the VMCPU.
7631	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7632	* @thread EMT(pVCpu)
7633	*/
7634	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7635	{
7636	NOREF(fGlobalInit);
7637
7638	switch (enmEvent)
7639	{
7640	case RTTHREADCTXEVENT_PREEMPTING:
7641	{
7642	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7643	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7644	VMCPU_ASSERT_EMT(pVCpu);
7645
7646	PVM pVM = pVCpu->CTX_SUFF(pVM);
7647	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7648
7649	/* No longjmps (logger flushes, locks) in this fragile context. */
7650	VMMRZCallRing3Disable(pVCpu);
7651	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7652
7653	/*
7654	* Restore host-state (FPU, debug etc.)
7655	*/
7656	if (!pVCpu->hm.s.fLeaveDone)
7657	{
7658	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7659	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7660	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7661	pVCpu->hm.s.fLeaveDone = true;
7662	}
7663
7664	/* Leave HM context, takes care of local init (term). */
7665	int rc = HMR0LeaveCpu(pVCpu);
7666	AssertRC(rc); NOREF(rc);
7667
7668	/* Restore longjmp state. */
7669	VMMRZCallRing3Enable(pVCpu);
7670	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7671	break;
7672	}
7673
7674	case RTTHREADCTXEVENT_RESUMED:
7675	{
7676	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7677	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7678	VMCPU_ASSERT_EMT(pVCpu);
7679
7680	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7681	VMMRZCallRing3Disable(pVCpu);
7682	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7683
7684	/* Initialize the bare minimum state required for HM. This takes care of
7685	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7686	int rc = HMR0EnterCpu(pVCpu);
7687	AssertRC(rc);
7688	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7689
7690	/* Load the active VMCS as the current one. */
7691	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7692	{
7693	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7694	AssertRC(rc); NOREF(rc);
7695	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7696	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7697	}
7698	pVCpu->hm.s.fLeaveDone = false;
7699
7700	/* Restore longjmp state. */
7701	VMMRZCallRing3Enable(pVCpu);
7702	break;
7703	}
7704
7705	default:
7706	break;
7707	}
7708	}
7709
7710
7711	/**
7712	* Saves the host state in the VMCS host-state.
7713	* Sets up the VM-exit MSR-load area.
7714	*
7715	* The CPU state will be loaded from these fields on every successful VM-exit.
7716	*
7717	* @returns VBox status code.
7718	* @param pVM Pointer to the VM.
7719	* @param pVCpu Pointer to the VMCPU.
7720	*
7721	* @remarks No-long-jump zone!!!
7722	*/
7723	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7724	{
7725	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7726
7727	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7728	return VINF_SUCCESS;
7729
7730	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7731	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7732
7733	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7734	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7735
7736	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7737	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7738
7739	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7740	return rc;
7741	}
7742
7743
7744	/**
7745	* Saves the host state in the VMCS host-state.
7746	*
7747	* @returns VBox status code.
7748	* @param pVM Pointer to the VM.
7749	* @param pVCpu Pointer to the VMCPU.
7750	*
7751	* @remarks No-long-jump zone!!!
7752	*/
7753	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7754	{
7755	AssertPtr(pVM);
7756	AssertPtr(pVCpu);
7757
7758	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7759
7760	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7761	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7762	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7763	return hmR0VmxSaveHostState(pVM, pVCpu);
7764	}
7765
7766
7767	/**
7768	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7769	* loaded from these fields on every successful VM-entry.
7770	*
7771	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7772	* Sets up the VM-entry controls.
7773	* Sets up the appropriate VMX non-root function to execute guest code based on
7774	* the guest CPU mode.
7775	*
7776	* @returns VBox status code.
7777	* @param pVM Pointer to the VM.
7778	* @param pVCpu Pointer to the VMCPU.
7779	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7780	* out-of-sync. Make sure to update the required fields
7781	* before using them.
7782	*
7783	* @remarks No-long-jump zone!!!
7784	*/
7785	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7786	{
7787	AssertPtr(pVM);
7788	AssertPtr(pVCpu);
7789	AssertPtr(pMixedCtx);
7790	HMVMX_ASSERT_PREEMPT_SAFE();
7791
7792	#ifdef LOG_ENABLED
7793	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7794	* probably not initialized yet? Anyway this will do for now.
7795	*
7796	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7797	* interface and disable ring-3 calls when thread-context hooks are not
7798	* available. */
7799	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7800	VMMR0LogFlushDisable(pVCpu);
7801	#endif
7802
7803	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7804
7805	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7806
7807	/* Determine real-on-v86 mode. */
7808	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7809	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7810	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7811	{
7812	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7813	}
7814
7815	/*
7816	* Load the guest-state into the VMCS.
7817	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7818	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7819	*/
7820	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7821	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7822
7823	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7824	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7825	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7826
7827	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7828	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7829	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7830
7831	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7832	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7833
7834	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7835	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7836
7837	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7838	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7839	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7840
7841	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7842	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7843
7844	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7845	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7846
7847	/*
7848	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7849	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7850	*/
7851	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7852	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7853
7854	/* Clear any unused and reserved bits. */
7855	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7856
7857	#ifdef LOG_ENABLED
7858	/* Only reenable log-flushing if the caller has it enabled. */
7859	if (!fCallerDisabledLogFlush)
7860	VMMR0LogFlushEnable(pVCpu);
7861	#endif
7862
7863	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7864	return rc;
7865	}
7866
7867
7868	/**
7869	* Loads the state shared between the host and guest into the VMCS.
7870	*
7871	* @param pVM Pointer to the VM.
7872	* @param pVCpu Pointer to the VMCPU.
7873	* @param pCtx Pointer to the guest-CPU context.
7874	*
7875	* @remarks No-long-jump zone!!!
7876	*/
7877	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7878	{
7879	NOREF(pVM);
7880
7881	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7882	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7883
7884	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7885	{
7886	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7887	AssertRC(rc);
7888	}
7889
7890	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7891	{
7892	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7893	AssertRC(rc);
7894
7895	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7896	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7897	{
7898	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7899	AssertRC(rc);
7900	}
7901	}
7902
7903	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7904	{
7905	#if HC_ARCH_BITS == 64
7906	if (pVM->hm.s.fAllow64BitGuests)
7907	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7908	#endif
7909	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7910	}
7911
7912	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7913	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7914	}
7915
7916
7917	/**
7918	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7919	*
7920	* @param pVM Pointer to the VM.
7921	* @param pVCpu Pointer to the VMCPU.
7922	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7923	* out-of-sync. Make sure to update the required fields
7924	* before using them.
7925	*/
7926	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7927	{
7928	HMVMX_ASSERT_PREEMPT_SAFE();
7929
7930	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7931	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7932	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7933	#endif
7934
7935	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7936	{
7937	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7938	AssertRC(rc);
7939	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7940	}
7941	else if (HMCPU_CF_VALUE(pVCpu))
7942	{
7943	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7944	AssertRC(rc);
7945	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7946	}
7947
7948	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7949	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7950	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7951	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7952
7953	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7954	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7955	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7956	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7957	#endif
7958	}
7959
7960
7961	/**
7962	* Does the preparations before executing guest code in VT-x.
7963	*
7964	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7965	* recompiler. We must be cautious what we do here regarding committing
7966	* guest-state information into the VMCS assuming we assuredly execute the
7967	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7968	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7969	* so that the recompiler can (and should) use them when it resumes guest
7970	* execution. Otherwise such operations must be done when we can no longer
7971	* exit to ring-3.
7972	*
7973	* @returns Strict VBox status code.
7974	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7975	* have been disabled.
7976	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7977	* double-fault into the guest.
7978	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7979	*
7980	* @param pVM Pointer to the VM.
7981	* @param pVCpu Pointer to the VMCPU.
7982	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7983	* out-of-sync. Make sure to update the required fields
7984	* before using them.
7985	* @param pVmxTransient Pointer to the VMX transient structure.
7986	*/
7987	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7988	{
7989	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7990
7991	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7992	PGMRZDynMapFlushAutoSet(pVCpu);
7993	#endif
7994
7995	/* Check force flag actions that might require us to go back to ring-3. */
7996	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7997	if (rc != VINF_SUCCESS)
7998	return rc;
7999
8000	#ifndef IEM_VERIFICATION_MODE_FULL
8001	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8002	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8003	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8004	{
8005	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8006	RTGCPHYS GCPhysApicBase;
8007	GCPhysApicBase = pMixedCtx->msrApicBase;
8008	GCPhysApicBase &= PAGE_BASE_GC_MASK;
8009
8010	/* Unalias any existing mapping. */
8011	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8012	AssertRCReturn(rc, rc);
8013
8014	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8015	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8016	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8017	AssertRCReturn(rc, rc);
8018
8019	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8020	}
8021	#endif /* !IEM_VERIFICATION_MODE_FULL */
8022
8023	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
8024	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8025
8026	/*
8027	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8028	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8029	*/
8030	if (TRPMHasTrap(pVCpu))
8031	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8032	else if (!pVCpu->hm.s.Event.fPending)
8033	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8034
8035	/*
8036	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8037	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8038	*/
8039	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8040	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8041	{
8042	Assert(rc == VINF_EM_RESET);
8043	return rc;
8044	}
8045
8046	/*
8047	* No longjmps to ring-3 from this point on!!!
8048	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8049	* This also disables flushing of the R0-logger instance (if any).
8050	*/
8051	VMMRZCallRing3Disable(pVCpu);
8052
8053	/*
8054	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8055	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8056	*
8057	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8058	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8059	*
8060	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8061	* executing guest code.
8062	*/
8063	pVmxTransient->uEflags = ASMIntDisableFlags();
8064	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8065	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8066	{
8067	hmR0VmxClearEventVmcs(pVCpu);
8068	ASMSetFlags(pVmxTransient->uEflags);
8069	VMMRZCallRing3Enable(pVCpu);
8070	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8071	return VINF_EM_RAW_TO_R3;
8072	}
8073	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8074	{
8075	hmR0VmxClearEventVmcs(pVCpu);
8076	ASMSetFlags(pVmxTransient->uEflags);
8077	VMMRZCallRing3Enable(pVCpu);
8078	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8079	return VINF_EM_RAW_INTERRUPT;
8080	}
8081
8082	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8083	pVCpu->hm.s.Event.fPending = false;
8084
8085	return VINF_SUCCESS;
8086	}
8087
8088
8089	/**
8090	* Prepares to run guest code in VT-x and we've committed to doing so. This
8091	* means there is no backing out to ring-3 or anywhere else at this
8092	* point.
8093	*
8094	* @param pVM Pointer to the VM.
8095	* @param pVCpu Pointer to the VMCPU.
8096	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8097	* out-of-sync. Make sure to update the required fields
8098	* before using them.
8099	* @param pVmxTransient Pointer to the VMX transient structure.
8100	*
8101	* @remarks Called with preemption disabled.
8102	* @remarks No-long-jump zone!!!
8103	*/
8104	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8105	{
8106	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8107	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8108	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8109
8110	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8111	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8112
8113	/*
8114	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8115	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8116	* Reload only the necessary state, the assertion will catch if other parts of the code
8117	* change.
8118	*/
8119	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8120	{
8121	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8122	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8123	}
8124
8125	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8126	if (!CPUMIsGuestFPUStateActive(pVCpu))
8127	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8128	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8129	#endif
8130
8131	if ( pVCpu->hm.s.fUseGuestFpu
8132	&& !CPUMIsGuestFPUStateActive(pVCpu))
8133	{
8134	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8135	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8136	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8137	}
8138
8139	/*
8140	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8141	*/
8142	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8143	&& pVCpu->hm.s.vmx.cMsrs > 0)
8144	{
8145	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8146	}
8147
8148	/*
8149	* Load the host state bits as we may've been preempted (only happens when
8150	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8151	*/
8152	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8153	{
8154	/* This ASSUMES that pfnStartVM has been set up already. */
8155	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8156	AssertRC(rc);
8157	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8158	}
8159	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8160
8161	/*
8162	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8163	*/
8164	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8165	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8166	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8167
8168	/* Store status of the shared guest-host state at the time of VM-entry. */
8169	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8170	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8171	{
8172	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8173	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8174	}
8175	else
8176	#endif
8177	{
8178	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8179	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8180	}
8181	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8182
8183	/*
8184	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8185	*/
8186	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8187	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8188
8189	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8190	RTCPUID idCurrentCpu = pCpu->idCpu;
8191	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8192	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8193	{
8194	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8195	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8196	}
8197
8198	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8199	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8200	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8201	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8202
8203	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8204
8205	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8206	to start executing. */
8207
8208	/*
8209	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8210	*/
8211	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8212	{
8213	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8214	{
8215	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8216	AssertRC(rc2);
8217	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8218	bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8219	true /* fUpdateHostMsr */);
8220	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8221	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8222	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8223	}
8224	else
8225	{
8226	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8227	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8228	}
8229	}
8230	#ifdef VBOX_STRICT
8231	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8232	#endif
8233	}
8234
8235
8236	/**
8237	* Performs some essential restoration of state after running guest code in
8238	* VT-x.
8239	*
8240	* @param pVM Pointer to the VM.
8241	* @param pVCpu Pointer to the VMCPU.
8242	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8243	* out-of-sync. Make sure to update the required fields
8244	* before using them.
8245	* @param pVmxTransient Pointer to the VMX transient structure.
8246	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8247	*
8248	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8249	*
8250	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8251	* unconditionally when it is safe to do so.
8252	*/
8253	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8254	{
8255	NOREF(pVM);
8256
8257	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8258
8259	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8260	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8261	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8262	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8263	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8264
8265	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8266	{
8267	/** @todo Find a way to fix hardcoding a guestimate. */
8268	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8269	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8270	}
8271
8272	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8273	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8274	Assert(!(ASMGetFlags() & X86_EFL_IF));
8275	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8276
8277	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8278	if (CPUMIsGuestFPUStateActive(pVCpu))
8279	{
8280	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8281	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8282	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8283	}
8284	#endif
8285
8286	#if HC_ARCH_BITS == 64
8287	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8288	#endif
8289	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8290	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8291	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8292
8293	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8294	uint32_t uExitReason;
8295	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8296	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8297	AssertRC(rc);
8298	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8299	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8300
8301	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8302	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8303	{
8304	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8305	pVmxTransient->fVMEntryFailed));
8306	return;
8307	}
8308
8309	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8310	{
8311	/* Update the guest interruptibility-state from the VMCS. */
8312	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8313	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8314	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8315	AssertRC(rc);
8316	#endif
8317	/*
8318	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8319	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8320	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8321	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8322	*/
8323	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8324	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8325	{
8326	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8327	AssertRC(rc);
8328	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8329	}
8330	}
8331	}
8332
8333
8334
8335	/**
8336	* Runs the guest code using VT-x the normal way.
8337	*
8338	* @returns VBox status code.
8339	* @param pVM Pointer to the VM.
8340	* @param pVCpu Pointer to the VMCPU.
8341	* @param pCtx Pointer to the guest-CPU context.
8342	*
8343	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8344	*/
8345	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8346	{
8347	VMXTRANSIENT VmxTransient;
8348	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8349	int rc = VERR_INTERNAL_ERROR_5;
8350	uint32_t cLoops = 0;
8351
8352	for (;; cLoops++)
8353	{
8354	Assert(!HMR0SuspendPending());
8355	HMVMX_ASSERT_CPU_SAFE();
8356
8357	/* Preparatory work for running guest code, this may force us to return
8358	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8359	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8360	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8361	if (rc != VINF_SUCCESS)
8362	break;
8363
8364	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8365	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8366	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8367
8368	/* Restore any residual host-state and save any bits shared between host
8369	and guest into the guest-CPU state. Re-enables interrupts! */
8370	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8371
8372	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8373	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8374	{
8375	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8376	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8377	return rc;
8378	}
8379
8380	/* Handle the VM-exit. */
8381	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8383	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8384	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8385	HMVMX_START_EXIT_DISPATCH_PROF();
8386	#ifdef HMVMX_USE_FUNCTION_TABLE
8387	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8388	#else
8389	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8390	#endif
8391	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8392	if (rc != VINF_SUCCESS)
8393	break;
8394	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8395	{
8396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8397	rc = VINF_EM_RAW_INTERRUPT;
8398	break;
8399	}
8400	}
8401
8402	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8403	return rc;
8404	}
8405
8406
8407	/**
8408	* Single steps guest code using VT-x.
8409	*
8410	* @returns VBox status code.
8411	* @param pVM Pointer to the VM.
8412	* @param pVCpu Pointer to the VMCPU.
8413	* @param pCtx Pointer to the guest-CPU context.
8414	*
8415	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8416	*/
8417	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8418	{
8419	VMXTRANSIENT VmxTransient;
8420	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8421	int rc = VERR_INTERNAL_ERROR_5;
8422	uint32_t cLoops = 0;
8423	uint16_t uCsStart = pCtx->cs.Sel;
8424	uint64_t uRipStart = pCtx->rip;
8425
8426	for (;; cLoops++)
8427	{
8428	Assert(!HMR0SuspendPending());
8429	HMVMX_ASSERT_CPU_SAFE();
8430
8431	/* Preparatory work for running guest code, this may force us to return
8432	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8433	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8434	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8435	if (rc != VINF_SUCCESS)
8436	break;
8437
8438	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8439	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8440	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8441
8442	/* Restore any residual host-state and save any bits shared between host
8443	and guest into the guest-CPU state. Re-enables interrupts! */
8444	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8445
8446	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8447	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8448	{
8449	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8450	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8451	return rc;
8452	}
8453
8454	/* Handle the VM-exit. */
8455	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8456	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8457	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8458	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8459	HMVMX_START_EXIT_DISPATCH_PROF();
8460	#ifdef HMVMX_USE_FUNCTION_TABLE
8461	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8462	#else
8463	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8464	#endif
8465	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8466	if (rc != VINF_SUCCESS)
8467	break;
8468	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8469	{
8470	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8471	rc = VINF_EM_RAW_INTERRUPT;
8472	break;
8473	}
8474
8475	/*
8476	* Did the RIP change, if so, consider it a single step.
8477	* Otherwise, make sure one of the TFs gets set.
8478	*/
8479	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8480	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8481	AssertRCReturn(rc2, rc2);
8482	if ( pCtx->rip != uRipStart
8483	\|\| pCtx->cs.Sel != uCsStart)
8484	{
8485	rc = VINF_EM_DBG_STEPPED;
8486	break;
8487	}
8488	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8489	}
8490
8491	/*
8492	* Clear the X86_EFL_TF if necessary.
8493	*/
8494	if (pVCpu->hm.s.fClearTrapFlag)
8495	{
8496	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8497	AssertRCReturn(rc2, rc2);
8498	pVCpu->hm.s.fClearTrapFlag = false;
8499	pCtx->eflags.Bits.u1TF = 0;
8500	}
8501	/** @todo there seems to be issues with the resume flag when the monitor trap
8502	* flag is pending without being used. Seen early in bios init when
8503	* accessing APIC page in prot mode. */
8504
8505	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8506	return rc;
8507	}
8508
8509
8510	/**
8511	* Runs the guest code using VT-x.
8512	*
8513	* @returns VBox status code.
8514	* @param pVM Pointer to the VM.
8515	* @param pVCpu Pointer to the VMCPU.
8516	* @param pCtx Pointer to the guest-CPU context.
8517	*/
8518	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8519	{
8520	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8521	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8522	HMVMX_ASSERT_PREEMPT_SAFE();
8523
8524	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8525
8526	int rc;
8527	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8528	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8529	else
8530	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8531
8532	if (rc == VERR_EM_INTERPRETER)
8533	rc = VINF_EM_RAW_EMULATE_INSTR;
8534	else if (rc == VINF_EM_RESET)
8535	rc = VINF_EM_TRIPLE_FAULT;
8536
8537	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8538	if (RT_FAILURE(rc2))
8539	{
8540	pVCpu->hm.s.u32HMError = rc;
8541	rc = rc2;
8542	}
8543	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8544	return rc;
8545	}
8546
8547
8548	#ifndef HMVMX_USE_FUNCTION_TABLE
8549	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8550	{
8551	#ifdef DEBUG_ramshankar
8552	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8553	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8554	#endif
8555	int rc;
8556	switch (rcReason)
8557	{
8558	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8559	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8560	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8561	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8562	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8563	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8564	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8565	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8566	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8567	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8568	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8569	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8570	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8571	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8572	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8573	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8574	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8575	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8576	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8577	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8578	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8579	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8580	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8581	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8582	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8583	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8584	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8585	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8586	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8587	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8588	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8589	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8590	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8591
8592	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8593	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8594	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8595	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8596	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8597	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8598	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8599	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8600	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8601
8602	case VMX_EXIT_VMCALL:
8603	case VMX_EXIT_VMCLEAR:
8604	case VMX_EXIT_VMLAUNCH:
8605	case VMX_EXIT_VMPTRLD:
8606	case VMX_EXIT_VMPTRST:
8607	case VMX_EXIT_VMREAD:
8608	case VMX_EXIT_VMRESUME:
8609	case VMX_EXIT_VMWRITE:
8610	case VMX_EXIT_VMXOFF:
8611	case VMX_EXIT_VMXON:
8612	case VMX_EXIT_INVEPT:
8613	case VMX_EXIT_INVVPID:
8614	case VMX_EXIT_VMFUNC:
8615	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8616	break;
8617	default:
8618	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8619	break;
8620	}
8621	return rc;
8622	}
8623	#endif
8624
8625	#ifdef DEBUG
8626	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8627	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8628	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8629
8630	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8631	do \
8632	{ \
8633	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8634	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8635	} while (0)
8636
8637	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8638	do { \
8639	AssertPtr(pVCpu); \
8640	AssertPtr(pMixedCtx); \
8641	AssertPtr(pVmxTransient); \
8642	Assert(pVmxTransient->fVMEntryFailed == false); \
8643	Assert(ASMIntAreEnabled()); \
8644	HMVMX_ASSERT_PREEMPT_SAFE(); \
8645	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8646	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8647	HMVMX_ASSERT_PREEMPT_SAFE(); \
8648	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8649	HMVMX_ASSERT_PREEMPT_CPUID(); \
8650	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8651	} while (0)
8652
8653	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8654	do { \
8655	Log4Func(("\n")); \
8656	} while (0)
8657	#else /* Release builds */
8658	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8659	do { \
8660	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8661	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8662	} while (0)
8663	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8664	#endif
8665
8666
8667	/**
8668	* Advances the guest RIP after reading it from the VMCS.
8669	*
8670	* @returns VBox status code.
8671	* @param pVCpu Pointer to the VMCPU.
8672	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8673	* out-of-sync. Make sure to update the required fields
8674	* before using them.
8675	* @param pVmxTransient Pointer to the VMX transient structure.
8676	*
8677	* @remarks No-long-jump zone!!!
8678	*/
8679	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8680	{
8681	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8682	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8683	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8684	AssertRCReturn(rc, rc);
8685
8686	pMixedCtx->rip += pVmxTransient->cbInstr;
8687	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8688
8689	/*
8690	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8691	* pending debug exception field as it takes care of priority of events.
8692	*
8693	* See Intel spec. 32.2.1 "Debug Exceptions".
8694	*/
8695	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8696
8697	return rc;
8698	}
8699
8700
8701	/**
8702	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8703	* and update error record fields accordingly.
8704	*
8705	* @return VMX_IGS_* return codes.
8706	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8707	* wrong with the guest state.
8708	*
8709	* @param pVM Pointer to the VM.
8710	* @param pVCpu Pointer to the VMCPU.
8711	* @param pCtx Pointer to the guest-CPU state.
8712	*/
8713	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8714	{
8715	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8716	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8717	uError = (err); \
8718	break; \
8719	} else do { } while (0)
8720
8721	int rc;
8722	uint32_t uError = VMX_IGS_ERROR;
8723	uint32_t u32Val;
8724	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8725
8726	do
8727	{
8728	/*
8729	* CR0.
8730	*/
8731	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8732	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8733	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8734	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8735	if (fUnrestrictedGuest)
8736	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8737
8738	uint32_t u32GuestCR0;
8739	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8740	AssertRCBreak(rc);
8741	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8742	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8743	if ( !fUnrestrictedGuest
8744	&& (u32GuestCR0 & X86_CR0_PG)
8745	&& !(u32GuestCR0 & X86_CR0_PE))
8746	{
8747	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8748	}
8749
8750	/*
8751	* CR4.
8752	*/
8753	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8754	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8755
8756	uint32_t u32GuestCR4;
8757	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8758	AssertRCBreak(rc);
8759	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8760	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8761
8762	/*
8763	* IA32_DEBUGCTL MSR.
8764	*/
8765	uint64_t u64Val;
8766	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8767	AssertRCBreak(rc);
8768	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8769	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8770	{
8771	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8772	}
8773	uint64_t u64DebugCtlMsr = u64Val;
8774
8775	#ifdef VBOX_STRICT
8776	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8777	AssertRCBreak(rc);
8778	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8779	#endif
8780	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8781
8782	/*
8783	* RIP and RFLAGS.
8784	*/
8785	uint32_t u32Eflags;
8786	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8787	if (HMVMX_IS_64BIT_HOST_MODE())
8788	{
8789	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8790	AssertRCBreak(rc);
8791	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8792	if ( !fLongModeGuest
8793	\|\| !pCtx->cs.Attr.n.u1Long)
8794	{
8795	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8796	}
8797	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8798	* must be identical if the "IA32e mode guest" VM-entry control is 1
8799	* and CS.L is 1. No check applies if the CPU supports 64
8800	* linear-address bits. */
8801
8802	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8803	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8804	AssertRCBreak(rc);
8805	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8806	VMX_IGS_RFLAGS_RESERVED);
8807	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8808	u32Eflags = u64Val;
8809	}
8810	else
8811	#endif
8812	{
8813	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8814	AssertRCBreak(rc);
8815	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8816	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8817	}
8818
8819	if ( fLongModeGuest
8820	\|\| ( fUnrestrictedGuest
8821	&& !(u32GuestCR0 & X86_CR0_PE)))
8822	{
8823	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8824	}
8825
8826	uint32_t u32EntryInfo;
8827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8828	AssertRCBreak(rc);
8829	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8830	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8831	{
8832	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8833	}
8834
8835	/*
8836	* 64-bit checks.
8837	*/
8838	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8839	if (HMVMX_IS_64BIT_HOST_MODE())
8840	{
8841	if ( fLongModeGuest
8842	&& !fUnrestrictedGuest)
8843	{
8844	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8845	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8846	}
8847
8848	if ( !fLongModeGuest
8849	&& (u32GuestCR4 & X86_CR4_PCIDE))
8850	{
8851	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8852	}
8853
8854	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8855	* 51:32 beyond the processor's physical-address width are 0. */
8856
8857	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8858	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8859	{
8860	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8861	}
8862
8863	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8864	AssertRCBreak(rc);
8865	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8866
8867	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8868	AssertRCBreak(rc);
8869	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8870	}
8871	#endif
8872
8873	/*
8874	* PERF_GLOBAL MSR.
8875	*/
8876	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8877	{
8878	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8879	AssertRCBreak(rc);
8880	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8881	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8882	}
8883
8884	/*
8885	* PAT MSR.
8886	*/
8887	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8888	{
8889	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8890	AssertRCBreak(rc);
8891	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8892	for (unsigned i = 0; i < 8; i++)
8893	{
8894	uint8_t u8Val = (u64Val & 0x7);
8895	if ( u8Val != 0 /* UC */
8896	\|\| u8Val != 1 /* WC */
8897	\|\| u8Val != 4 /* WT */
8898	\|\| u8Val != 5 /* WP */
8899	\|\| u8Val != 6 /* WB */
8900	\|\| u8Val != 7 /* UC- */)
8901	{
8902	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8903	}
8904	u64Val >>= 3;
8905	}
8906	}
8907
8908	/*
8909	* EFER MSR.
8910	*/
8911	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8912	{
8913	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8914	AssertRCBreak(rc);
8915	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8916	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8917	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8918	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8919	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8920	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8921	}
8922
8923	/*
8924	* Segment registers.
8925	*/
8926	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8927	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8928	if (!(u32Eflags & X86_EFL_VM))
8929	{
8930	/* CS */
8931	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8932	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8933	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8934	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8935	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8936	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8937	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8938	/* CS cannot be loaded with NULL in protected mode. */
8939	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8940	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8941	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8942	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8943	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8944	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8945	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8946	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8947	else
8948	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8949
8950	/* SS */
8951	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8952	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8953	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8954	if ( !(pCtx->cr0 & X86_CR0_PE)
8955	\|\| pCtx->cs.Attr.n.u4Type == 3)
8956	{
8957	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8958	}
8959	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8960	{
8961	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8962	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8963	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8964	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8965	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8966	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8967	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8968	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8969	}
8970
8971	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8972	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8973	{
8974	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8975	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8976	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8977	\|\| pCtx->ds.Attr.n.u4Type > 11
8978	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8979	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8980	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8981	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8982	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8983	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8984	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8985	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8986	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8987	}
8988	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8989	{
8990	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8991	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8992	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8993	\|\| pCtx->es.Attr.n.u4Type > 11
8994	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8995	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8996	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8997	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8998	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8999	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9000	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9001	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9002	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9003	}
9004	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9005	{
9006	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9007	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9008	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9009	\|\| pCtx->fs.Attr.n.u4Type > 11
9010	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9011	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9012	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9013	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9014	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9015	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9016	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9017	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9018	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9019	}
9020	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9021	{
9022	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9023	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9024	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9025	\|\| pCtx->gs.Attr.n.u4Type > 11
9026	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9027	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9028	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9029	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9030	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9031	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9032	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9033	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9034	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9035	}
9036	/* 64-bit capable CPUs. */
9037	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9038	if (HMVMX_IS_64BIT_HOST_MODE())
9039	{
9040	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9041	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9042	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9043	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9044	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9045	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9046	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9047	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9048	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9049	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9050	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9051	}
9052	#endif
9053	}
9054	else
9055	{
9056	/* V86 mode checks. */
9057	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9058	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9059	{
9060	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9061	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9062	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9063	}
9064	else
9065	{
9066	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9067	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9068	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9069	}
9070
9071	/* CS */
9072	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9073	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9074	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9075	/* SS */
9076	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9077	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9078	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9079	/* DS */
9080	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9081	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9082	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9083	/* ES */
9084	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9085	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9086	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9087	/* FS */
9088	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9089	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9090	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9091	/* GS */
9092	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9093	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9094	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9095	/* 64-bit capable CPUs. */
9096	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9097	if (HMVMX_IS_64BIT_HOST_MODE())
9098	{
9099	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9100	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9101	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9102	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9103	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9104	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9105	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9106	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9107	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9108	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9109	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9110	}
9111	#endif
9112	}
9113
9114	/*
9115	* TR.
9116	*/
9117	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9118	/* 64-bit capable CPUs. */
9119	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9120	if (HMVMX_IS_64BIT_HOST_MODE())
9121	{
9122	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9123	}
9124	#endif
9125	if (fLongModeGuest)
9126	{
9127	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9128	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9129	}
9130	else
9131	{
9132	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9133	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9134	VMX_IGS_TR_ATTR_TYPE_INVALID);
9135	}
9136	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9137	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9138	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9139	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9140	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9141	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9142	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9143	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9144
9145	/*
9146	* GDTR and IDTR.
9147	*/
9148	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9149	if (HMVMX_IS_64BIT_HOST_MODE())
9150	{
9151	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9152	AssertRCBreak(rc);
9153	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9154
9155	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9156	AssertRCBreak(rc);
9157	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9158	}
9159	#endif
9160
9161	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9162	AssertRCBreak(rc);
9163	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9164
9165	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9166	AssertRCBreak(rc);
9167	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9168
9169	/*
9170	* Guest Non-Register State.
9171	*/
9172	/* Activity State. */
9173	uint32_t u32ActivityState;
9174	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9175	AssertRCBreak(rc);
9176	HMVMX_CHECK_BREAK( !u32ActivityState
9177	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9178	VMX_IGS_ACTIVITY_STATE_INVALID);
9179	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9180	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9181	uint32_t u32IntrState;
9182	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9183	AssertRCBreak(rc);
9184	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9185	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9186	{
9187	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9188	}
9189
9190	/** @todo Activity state and injecting interrupts. Left as a todo since we
9191	* currently don't use activity states but ACTIVE. */
9192
9193	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9194	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9195
9196	/* Guest interruptibility-state. */
9197	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9198	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9199	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9200	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9201	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9202	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9203	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9204	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9205	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9206	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9207	{
9208	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9209	{
9210	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9211	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9212	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9213	}
9214	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9215	{
9216	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9217	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9218	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9219	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9220	}
9221	}
9222	/** @todo Assumes the processor is not in SMM. */
9223	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9224	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9225	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9226	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9227	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9228	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9229	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9230	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9231	{
9232	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9233	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9234	}
9235
9236	/* Pending debug exceptions. */
9237	if (HMVMX_IS_64BIT_HOST_MODE())
9238	{
9239	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9240	AssertRCBreak(rc);
9241	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9242	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9243	u32Val = u64Val; /* For pending debug exceptions checks below. */
9244	}
9245	else
9246	{
9247	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9248	AssertRCBreak(rc);
9249	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9250	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9251	}
9252
9253	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9254	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9255	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9256	{
9257	if ( (u32Eflags & X86_EFL_TF)
9258	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9259	{
9260	/* Bit 14 is PendingDebug.BS. */
9261	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9262	}
9263	if ( !(u32Eflags & X86_EFL_TF)
9264	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9265	{
9266	/* Bit 14 is PendingDebug.BS. */
9267	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9268	}
9269	}
9270
9271	/* VMCS link pointer. */
9272	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9273	AssertRCBreak(rc);
9274	if (u64Val != UINT64_C(0xffffffffffffffff))
9275	{
9276	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9277	/** @todo Bits beyond the processor's physical-address width MBZ. */
9278	/** @todo 32-bit located in memory referenced by value of this field (as a
9279	* physical address) must contain the processor's VMCS revision ID. */
9280	/** @todo SMM checks. */
9281	}
9282
9283	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9284	* not using Nested Paging? */
9285	if ( pVM->hm.s.fNestedPaging
9286	&& !fLongModeGuest
9287	&& CPUMIsGuestInPAEModeEx(pCtx))
9288	{
9289	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9290	AssertRCBreak(rc);
9291	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9292
9293	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9294	AssertRCBreak(rc);
9295	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9296
9297	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9298	AssertRCBreak(rc);
9299	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9300
9301	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9302	AssertRCBreak(rc);
9303	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9304	}
9305
9306	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9307	if (uError == VMX_IGS_ERROR)
9308	uError = VMX_IGS_REASON_NOT_FOUND;
9309	} while (0);
9310
9311	pVCpu->hm.s.u32HMError = uError;
9312	return uError;
9313
9314	#undef HMVMX_ERROR_BREAK
9315	#undef HMVMX_CHECK_BREAK
9316	}
9317
9318	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9319	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9320	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9321
9322	/** @name VM-exit handlers.
9323	* @{
9324	*/
9325
9326	/**
9327	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9328	*/
9329	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9330	{
9331	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9332	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9333	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9334	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9335	return VINF_SUCCESS;
9336	return VINF_EM_RAW_INTERRUPT;
9337	}
9338
9339
9340	/**
9341	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9342	*/
9343	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9344	{
9345	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9346	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9347
9348	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9349	AssertRCReturn(rc, rc);
9350
9351	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9352	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9353	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9354	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9355
9356	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9357	{
9358	/*
9359	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9360	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9361	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9362	*
9363	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9364	*/
9365	VMXDispatchHostNmi();
9366	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9367	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9368	return VINF_SUCCESS;
9369	}
9370
9371	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9372	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9373	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9374	{
9375	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9376	return VINF_SUCCESS;
9377	}
9378	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9379	{
9380	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9381	return rc;
9382	}
9383
9384	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9385	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9386	switch (uIntType)
9387	{
9388	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9389	Assert(uVector == X86_XCPT_DB);
9390	/* no break */
9391	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9392	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9393	/* no break */
9394	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9395	{
9396	switch (uVector)
9397	{
9398	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9399	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9400	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9401	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9402	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9403	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9404	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9405	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9406	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9407	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9408	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9409	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9410	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9411	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9412	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9413	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9414	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9415	#endif
9416	default:
9417	{
9418	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9419	AssertRCReturn(rc, rc);
9420
9421	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9422	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9423	{
9424	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9425	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9426	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9427
9428	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9429	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9430	AssertRCReturn(rc, rc);
9431	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9432	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9433	0 /* GCPtrFaultAddress */);
9434	AssertRCReturn(rc, rc);
9435	}
9436	else
9437	{
9438	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9439	pVCpu->hm.s.u32HMError = uVector;
9440	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9441	}
9442	break;
9443	}
9444	}
9445	break;
9446	}
9447
9448	default:
9449	{
9450	pVCpu->hm.s.u32HMError = uExitIntInfo;
9451	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9452	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9453	break;
9454	}
9455	}
9456	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9457	return rc;
9458	}
9459
9460
9461	/**
9462	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9463	*/
9464	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9465	{
9466	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9467
9468	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9469	hmR0VmxClearIntWindowExitVmcs(pVCpu);
9470
9471	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9473	return VINF_SUCCESS;
9474	}
9475
9476
9477	/**
9478	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9479	*/
9480	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9481	{
9482	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9483	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9484	HMVMX_RETURN_UNEXPECTED_EXIT();
9485	}
9486
9487
9488	/**
9489	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9490	*/
9491	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9492	{
9493	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9494	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9495	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9496	}
9497
9498
9499	/**
9500	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9501	*/
9502	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9503	{
9504	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9506	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9507	}
9508
9509
9510	/**
9511	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9512	*/
9513	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9514	{
9515	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9516	PVM pVM = pVCpu->CTX_SUFF(pVM);
9517	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9518	if (RT_LIKELY(rc == VINF_SUCCESS))
9519	{
9520	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9521	Assert(pVmxTransient->cbInstr == 2);
9522	}
9523	else
9524	{
9525	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9526	rc = VERR_EM_INTERPRETER;
9527	}
9528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9529	return rc;
9530	}
9531
9532
9533	/**
9534	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9535	*/
9536	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9537	{
9538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9539	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9540	AssertRCReturn(rc, rc);
9541
9542	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9543	return VINF_EM_RAW_EMULATE_INSTR;
9544
9545	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9546	HMVMX_RETURN_UNEXPECTED_EXIT();
9547	}
9548
9549
9550	/**
9551	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9552	*/
9553	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9554	{
9555	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9556	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9557	AssertRCReturn(rc, rc);
9558
9559	PVM pVM = pVCpu->CTX_SUFF(pVM);
9560	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9561	if (RT_LIKELY(rc == VINF_SUCCESS))
9562	{
9563	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9564	Assert(pVmxTransient->cbInstr == 2);
9565	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9566	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9567	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9568	}
9569	else
9570	{
9571	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9572	rc = VERR_EM_INTERPRETER;
9573	}
9574	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9575	return rc;
9576	}
9577
9578
9579	/**
9580	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9581	*/
9582	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9583	{
9584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9585	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9586	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9587	AssertRCReturn(rc, rc);
9588
9589	PVM pVM = pVCpu->CTX_SUFF(pVM);
9590	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9591	if (RT_LIKELY(rc == VINF_SUCCESS))
9592	{
9593	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9594	Assert(pVmxTransient->cbInstr == 3);
9595	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9596	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9597	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9598	}
9599	else
9600	{
9601	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9602	rc = VERR_EM_INTERPRETER;
9603	}
9604	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9605	return rc;
9606	}
9607
9608
9609	/**
9610	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9611	*/
9612	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9613	{
9614	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9615	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9616	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9617	AssertRCReturn(rc, rc);
9618
9619	PVM pVM = pVCpu->CTX_SUFF(pVM);
9620	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9621	if (RT_LIKELY(rc == VINF_SUCCESS))
9622	{
9623	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9624	Assert(pVmxTransient->cbInstr == 2);
9625	}
9626	else
9627	{
9628	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9629	rc = VERR_EM_INTERPRETER;
9630	}
9631	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9632	return rc;
9633	}
9634
9635
9636	/**
9637	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9638	*/
9639	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9640	{
9641	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9642	PVM pVM = pVCpu->CTX_SUFF(pVM);
9643	Assert(!pVM->hm.s.fNestedPaging);
9644
9645	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9646	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9647	AssertRCReturn(rc, rc);
9648
9649	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9650	rc = VBOXSTRICTRC_VAL(rc2);
9651	if (RT_LIKELY(rc == VINF_SUCCESS))
9652	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9653	else
9654	{
9655	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9656	pVmxTransient->uExitQualification, rc));
9657	}
9658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9659	return rc;
9660	}
9661
9662
9663	/**
9664	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9665	*/
9666	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9667	{
9668	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9669	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9670	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9671	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9672	AssertRCReturn(rc, rc);
9673
9674	PVM pVM = pVCpu->CTX_SUFF(pVM);
9675	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9676	if (RT_LIKELY(rc == VINF_SUCCESS))
9677	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9678	else
9679	{
9680	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9681	rc = VERR_EM_INTERPRETER;
9682	}
9683	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9684	return rc;
9685	}
9686
9687
9688	/**
9689	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9690	*/
9691	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9692	{
9693	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9694	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9695	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9696	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9697	AssertRCReturn(rc, rc);
9698
9699	PVM pVM = pVCpu->CTX_SUFF(pVM);
9700	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9701	rc = VBOXSTRICTRC_VAL(rc2);
9702	if (RT_LIKELY( rc == VINF_SUCCESS
9703	\|\| rc == VINF_EM_HALT))
9704	{
9705	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9706	AssertRCReturn(rc3, rc3);
9707
9708	if ( rc == VINF_EM_HALT
9709	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9710	{
9711	rc = VINF_SUCCESS;
9712	}
9713	}
9714	else
9715	{
9716	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9717	rc = VERR_EM_INTERPRETER;
9718	}
9719	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9720	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9721	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9722	return rc;
9723	}
9724
9725
9726	/**
9727	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9728	*/
9729	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9730	{
9731	/*
9732	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9733	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9734	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9735	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9736	*/
9737	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9738	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9739	HMVMX_RETURN_UNEXPECTED_EXIT();
9740	}
9741
9742
9743	/**
9744	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9745	*/
9746	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9747	{
9748	/*
9749	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9750	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9751	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9752	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9753	*/
9754	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9755	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9756	HMVMX_RETURN_UNEXPECTED_EXIT();
9757	}
9758
9759
9760	/**
9761	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9762	*/
9763	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9764	{
9765	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9766	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9767	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9768	HMVMX_RETURN_UNEXPECTED_EXIT();
9769	}
9770
9771
9772	/**
9773	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9774	*/
9775	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9776	{
9777	/*
9778	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9779	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9780	* See Intel spec. 25.3 "Other Causes of VM-exits".
9781	*/
9782	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9783	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9784	HMVMX_RETURN_UNEXPECTED_EXIT();
9785	}
9786
9787
9788	/**
9789	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9790	* VM-exit.
9791	*/
9792	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9793	{
9794	/*
9795	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9796	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9797	*
9798	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9799	* See Intel spec. "23.8 Restrictions on VMX operation".
9800	*/
9801	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9802	return VINF_SUCCESS;
9803	}
9804
9805
9806	/**
9807	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9808	* VM-exit.
9809	*/
9810	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9811	{
9812	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9813	return VINF_EM_RESET;
9814	}
9815
9816
9817	/**
9818	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9819	*/
9820	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9821	{
9822	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9823	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9824	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9825	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9826	AssertRCReturn(rc, rc);
9827
9828	pMixedCtx->rip++;
9829	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9830	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9831	rc = VINF_SUCCESS;
9832	else
9833	rc = VINF_EM_HALT;
9834
9835	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9836	return rc;
9837	}
9838
9839
9840	/**
9841	* VM-exit handler for instructions that result in a #UD exception delivered to
9842	* the guest.
9843	*/
9844	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9845	{
9846	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9847	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9848	return VINF_SUCCESS;
9849	}
9850
9851
9852	/**
9853	* VM-exit handler for expiry of the VMX preemption timer.
9854	*/
9855	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9856	{
9857	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9858
9859	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9860	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9861
9862	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9863	PVM pVM = pVCpu->CTX_SUFF(pVM);
9864	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9865	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9866	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9867	}
9868
9869
9870	/**
9871	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9872	*/
9873	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9874	{
9875	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9876
9877	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9878	/** @todo check if XSETBV is supported by the recompiler. */
9879	return VERR_EM_INTERPRETER;
9880	}
9881
9882
9883	/**
9884	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9885	*/
9886	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9887	{
9888	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9889
9890	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9891	/** @todo implement EMInterpretInvpcid() */
9892	return VERR_EM_INTERPRETER;
9893	}
9894
9895
9896	/**
9897	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9898	* Error VM-exit.
9899	*/
9900	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9901	{
9902	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9903	AssertRCReturn(rc, rc);
9904
9905	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9906	NOREF(uInvalidReason);
9907
9908	#ifdef VBOX_STRICT
9909	uint32_t uIntrState;
9910	HMVMXHCUINTREG uHCReg;
9911	uint64_t u64Val;
9912	uint32_t u32Val;
9913
9914	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9915	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9916	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9917	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9918	AssertRCReturn(rc, rc);
9919
9920	Log4(("uInvalidReason %u\n", uInvalidReason));
9921	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9922	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9923	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9924	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9925
9926	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9927	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9928	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9929	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9930	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9931	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9932	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9933	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9934	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9935	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9936	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9937	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9938	#else
9939	NOREF(pVmxTransient);
9940	#endif
9941
9942	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9943	return VERR_VMX_INVALID_GUEST_STATE;
9944	}
9945
9946
9947	/**
9948	* VM-exit handler for VM-entry failure due to an MSR-load
9949	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9950	*/
9951	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9952	{
9953	NOREF(pVmxTransient);
9954	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9955	HMVMX_RETURN_UNEXPECTED_EXIT();
9956	}
9957
9958
9959	/**
9960	* VM-exit handler for VM-entry failure due to a machine-check event
9961	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9962	*/
9963	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9964	{
9965	NOREF(pVmxTransient);
9966	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9967	HMVMX_RETURN_UNEXPECTED_EXIT();
9968	}
9969
9970
9971	/**
9972	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9973	* theory.
9974	*/
9975	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9976	{
9977	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9978	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9979	return VERR_VMX_UNDEFINED_EXIT_CODE;
9980	}
9981
9982
9983	/**
9984	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9985	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9986	* Conditional VM-exit.
9987	*/
9988	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9989	{
9990	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9991
9992	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9993	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9994	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9995	return VERR_EM_INTERPRETER;
9996	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9997	HMVMX_RETURN_UNEXPECTED_EXIT();
9998	}
9999
10000
10001	/**
10002	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10003	*/
10004	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10005	{
10006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10007
10008	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10010	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10011	return VERR_EM_INTERPRETER;
10012	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10013	HMVMX_RETURN_UNEXPECTED_EXIT();
10014	}
10015
10016
10017	/**
10018	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10019	*/
10020	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10021	{
10022	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10023
10024	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10025	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10026	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10027	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10028	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10029	{
10030	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10031	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10032	}
10033	AssertRCReturn(rc, rc);
10034	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10035
10036	#ifdef VBOX_STRICT
10037	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10038	{
10039	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10040	{
10041	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10042	HMVMX_RETURN_UNEXPECTED_EXIT();
10043	}
10044	# if HC_ARCH_BITS == 64
10045	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10046	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10047	{
10048	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10049	HMVMX_RETURN_UNEXPECTED_EXIT();
10050	}
10051	# endif
10052	}
10053	#endif
10054
10055	PVM pVM = pVCpu->CTX_SUFF(pVM);
10056	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10057	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
10058	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10059	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10060
10061	if (RT_LIKELY(rc == VINF_SUCCESS))
10062	{
10063	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10064	Assert(pVmxTransient->cbInstr == 2);
10065	}
10066	return rc;
10067	}
10068
10069
10070	/**
10071	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10072	*/
10073	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10074	{
10075	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10076	PVM pVM = pVCpu->CTX_SUFF(pVM);
10077	int rc = VINF_SUCCESS;
10078
10079	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10080	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10081	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10082	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10083	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10084	{
10085	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10086	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10087	}
10088	AssertRCReturn(rc, rc);
10089	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10090
10091	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10092	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10093	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10094
10095	if (RT_LIKELY(rc == VINF_SUCCESS))
10096	{
10097	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10098
10099	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10100	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10101	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10102	{
10103	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10104	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10105	EMInterpretWrmsr() changes it. */
10106	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10107	}
10108	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10109	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10110
10111	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10112	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10113	{
10114	switch (pMixedCtx->ecx)
10115	{
10116	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10117	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10118	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10119	case MSR_K8_FS_BASE: /* no break */
10120	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10121	default:
10122	{
10123	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10124	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10125	#if HC_ARCH_BITS == 64
10126	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10127	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10128	#endif
10129	break;
10130	}
10131	}
10132	}
10133	#ifdef VBOX_STRICT
10134	else
10135	{
10136	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10137	switch (pMixedCtx->ecx)
10138	{
10139	case MSR_IA32_SYSENTER_CS:
10140	case MSR_IA32_SYSENTER_EIP:
10141	case MSR_IA32_SYSENTER_ESP:
10142	case MSR_K8_FS_BASE:
10143	case MSR_K8_GS_BASE:
10144	{
10145	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10146	HMVMX_RETURN_UNEXPECTED_EXIT();
10147	}
10148
10149	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10150	default:
10151	{
10152	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10153	{
10154	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10155	pMixedCtx->ecx));
10156	HMVMX_RETURN_UNEXPECTED_EXIT();
10157	}
10158
10159	#if HC_ARCH_BITS == 64
10160	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10161	{
10162	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10163	HMVMX_RETURN_UNEXPECTED_EXIT();
10164	}
10165	#endif
10166	break;
10167	}
10168	}
10169	}
10170	#endif /* VBOX_STRICT */
10171	}
10172	return rc;
10173	}
10174
10175
10176	/**
10177	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10178	*/
10179	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10180	{
10181	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10182
10183	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10185	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10186	return VERR_EM_INTERPRETER;
10187	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10188	HMVMX_RETURN_UNEXPECTED_EXIT();
10189	}
10190
10191
10192	/**
10193	* VM-exit handler for when the TPR value is lowered below the specified
10194	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10195	*/
10196	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10197	{
10198	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10199	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10200
10201	/*
10202	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10203	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10204	* resume guest execution.
10205	*/
10206	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10207	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10208	return VINF_SUCCESS;
10209	}
10210
10211
10212	/**
10213	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10214	* VM-exit.
10215	*
10216	* @retval VINF_SUCCESS when guest execution can continue.
10217	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10218	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10219	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10220	* recompiler.
10221	*/
10222	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10223	{
10224	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10225	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10226	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10227	AssertRCReturn(rc, rc);
10228
10229	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10230	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10231	PVM pVM = pVCpu->CTX_SUFF(pVM);
10232	switch (uAccessType)
10233	{
10234	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10235	{
10236	#if 0
10237	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10238	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10239	#else
10240	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10241	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10242	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10243	#endif
10244	AssertRCReturn(rc, rc);
10245
10246	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10247	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10248	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10249	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10250
10251	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10252	{
10253	case 0: /* CR0 */
10254	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10255	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10256	break;
10257	case 2: /* CR2 */
10258	/* Nothing to do here, CR2 it's not part of the VMCS. */
10259	break;
10260	case 3: /* CR3 */
10261	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10262	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10263	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10264	break;
10265	case 4: /* CR4 */
10266	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10267	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10268	break;
10269	case 8: /* CR8 */
10270	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10271	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10272	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10273	break;
10274	default:
10275	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10276	break;
10277	}
10278
10279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10280	break;
10281	}
10282
10283	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10284	{
10285	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10286	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10287	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10288	AssertRCReturn(rc, rc);
10289	Assert( !pVM->hm.s.fNestedPaging
10290	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10291	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10292
10293	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10294	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10295	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10296
10297	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10298	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10299	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10300	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10302	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10303	break;
10304	}
10305
10306	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10307	{
10308	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10309	AssertRCReturn(rc, rc);
10310	rc = EMInterpretCLTS(pVM, pVCpu);
10311	AssertRCReturn(rc, rc);
10312	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10314	Log4(("CRX CLTS write rc=%d\n", rc));
10315	break;
10316	}
10317
10318	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10319	{
10320	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10321	AssertRCReturn(rc, rc);
10322	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10323	if (RT_LIKELY(rc == VINF_SUCCESS))
10324	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10325	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10326	Log4(("CRX LMSW write rc=%d\n", rc));
10327	break;
10328	}
10329
10330	default:
10331	{
10332	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10333	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10334	}
10335	}
10336
10337	/* Validate possible error codes. */
10338	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10339	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10340	if (RT_SUCCESS(rc))
10341	{
10342	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10343	AssertRCReturn(rc2, rc2);
10344	}
10345
10346	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10347	return rc;
10348	}
10349
10350
10351	/**
10352	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10353	* VM-exit.
10354	*/
10355	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10356	{
10357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10358	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10359
10360	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10361	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10362	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10363	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10364	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10365	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10366	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10367	AssertRCReturn(rc2, rc2);
10368
10369	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10370	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10371	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10372	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10373	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10374	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10375	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10376	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10377
10378	/* I/O operation lookup arrays. */
10379	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10380	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10381
10382	VBOXSTRICTRC rcStrict;
10383	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10384	const uint32_t cbInstr = pVmxTransient->cbInstr;
10385	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10386	PVM pVM = pVCpu->CTX_SUFF(pVM);
10387	if (fIOString)
10388	{
10389	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10390	/*
10391	* INS/OUTS - I/O String instruction.
10392	*
10393	* Use instruction-information if available, otherwise fall back on
10394	* interpreting the instruction.
10395	*/
10396	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10397	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10398	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10399	{
10400	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10401	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10402	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10403	AssertRCReturn(rc2, rc2);
10404	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10405	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10406	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10407	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10408	if (fIOWrite)
10409	{
10410	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10411	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10412	}
10413	else
10414	{
10415	/*
10416	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10417	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10418	* See Intel Instruction spec. for "INS".
10419	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10420	*/
10421	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10422	}
10423	}
10424	else
10425	{
10426	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10427	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10428	AssertRCReturn(rc2, rc2);
10429	rcStrict = IEMExecOne(pVCpu);
10430	}
10431	/** @todo IEM needs to be setting these flags somehow. */
10432	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10433	fUpdateRipAlready = true;
10434	#else
10435	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10436	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10437	if (RT_SUCCESS(rcStrict))
10438	{
10439	if (fIOWrite)
10440	{
10441	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10442	(DISCPUMODE)pDis->uAddrMode, cbValue);
10443	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10444	}
10445	else
10446	{
10447	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10448	(DISCPUMODE)pDis->uAddrMode, cbValue);
10449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10450	}
10451	}
10452	else
10453	{
10454	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10455	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10456	}
10457	#endif
10458	}
10459	else
10460	{
10461	/*
10462	* IN/OUT - I/O instruction.
10463	*/
10464	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10465	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10466	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10467	if (fIOWrite)
10468	{
10469	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10470	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10471	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10473	}
10474	else
10475	{
10476	uint32_t u32Result = 0;
10477	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10478	if (IOM_SUCCESS(rcStrict))
10479	{
10480	/* Save result of I/O IN instr. in AL/AX/EAX. */
10481	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10482	}
10483	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10484	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10486	}
10487	}
10488
10489	if (IOM_SUCCESS(rcStrict))
10490	{
10491	if (!fUpdateRipAlready)
10492	{
10493	pMixedCtx->rip += cbInstr;
10494	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10495	}
10496
10497	/*
10498	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10499	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10500	*/
10501	if (fIOString)
10502	{
10503	/** @todo Single-step for INS/OUTS with REP prefix? */
10504	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10505	}
10506	else if (fStepping)
10507	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10508
10509	/*
10510	* If any I/O breakpoints are armed, we need to check if one triggered
10511	* and take appropriate action.
10512	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10513	*/
10514	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10515	AssertRCReturn(rc2, rc2);
10516
10517	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10518	* execution engines about whether hyper BPs and such are pending. */
10519	uint32_t const uDr7 = pMixedCtx->dr[7];
10520	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10521	&& X86_DR7_ANY_RW_IO(uDr7)
10522	&& (pMixedCtx->cr4 & X86_CR4_DE))
10523	\|\| DBGFBpIsHwIoArmed(pVM)))
10524	{
10525	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10526
10527	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10528	VMMRZCallRing3Disable(pVCpu);
10529	HM_DISABLE_PREEMPT_IF_NEEDED();
10530
10531	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10532
10533	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10534	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10535	{
10536	/* Raise #DB. */
10537	if (fIsGuestDbgActive)
10538	ASMSetDR6(pMixedCtx->dr[6]);
10539	if (pMixedCtx->dr[7] != uDr7)
10540	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10541
10542	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10543	}
10544	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10545	else if ( rcStrict2 != VINF_SUCCESS
10546	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10547	rcStrict = rcStrict2;
10548
10549	HM_RESTORE_PREEMPT_IF_NEEDED();
10550	VMMRZCallRing3Enable(pVCpu);
10551	}
10552	}
10553
10554	#ifdef DEBUG
10555	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10556	Assert(!fIOWrite);
10557	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10558	Assert(fIOWrite);
10559	else
10560	{
10561	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10562	* statuses, that the VMM device and some others may return. See
10563	* IOM_SUCCESS() for guidance. */
10564	AssertMsg( RT_FAILURE(rcStrict)
10565	\|\| rcStrict == VINF_SUCCESS
10566	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10567	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10568	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10569	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10570	}
10571	#endif
10572
10573	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10574	return VBOXSTRICTRC_TODO(rcStrict);
10575	}
10576
10577
10578	/**
10579	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10580	* VM-exit.
10581	*/
10582	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10583	{
10584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10585
10586	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10587	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10588	AssertRCReturn(rc, rc);
10589	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10590	{
10591	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10592	AssertRCReturn(rc, rc);
10593	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10594	{
10595	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10596
10597	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10598	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10599	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10600	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10601	{
10602	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10603	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10604
10605	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10606	Assert(!pVCpu->hm.s.Event.fPending);
10607	pVCpu->hm.s.Event.fPending = true;
10608	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10609	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10610	AssertRCReturn(rc, rc);
10611	if (fErrorCodeValid)
10612	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10613	else
10614	pVCpu->hm.s.Event.u32ErrCode = 0;
10615	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10616	&& uVector == X86_XCPT_PF)
10617	{
10618	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10619	}
10620
10621	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10622	}
10623	}
10624	}
10625
10626	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10627	* emulation. */
10628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10629	return VERR_EM_INTERPRETER;
10630	}
10631
10632
10633	/**
10634	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10635	*/
10636	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10637	{
10638	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10639	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10640	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10641	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10642	AssertRCReturn(rc, rc);
10643	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10644	return VINF_EM_DBG_STEPPED;
10645	}
10646
10647
10648	/**
10649	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10650	*/
10651	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10652	{
10653	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10654
10655	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10656	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10657	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10658	return VINF_SUCCESS;
10659	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10660	return rc;
10661
10662	#if 0
10663	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10664	* just sync the whole thing. */
10665	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10666	#else
10667	/* Aggressive state sync. for now. */
10668	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10669	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10670	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10671	#endif
10672	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10673	AssertRCReturn(rc, rc);
10674
10675	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10676	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10677	switch (uAccessType)
10678	{
10679	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10680	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10681	{
10682	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10683	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10684	{
10685	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10686	}
10687
10688	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10689	GCPhys &= PAGE_BASE_GC_MASK;
10690	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10691	PVM pVM = pVCpu->CTX_SUFF(pVM);
10692	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10693	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10694
10695	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10696	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10697	CPUMCTX2CORE(pMixedCtx), GCPhys);
10698	rc = VBOXSTRICTRC_VAL(rc2);
10699	Log4(("ApicAccess rc=%d\n", rc));
10700	if ( rc == VINF_SUCCESS
10701	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10702	\|\| rc == VERR_PAGE_NOT_PRESENT)
10703	{
10704	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10705	\| HM_CHANGED_GUEST_RSP
10706	\| HM_CHANGED_GUEST_RFLAGS
10707	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10708	rc = VINF_SUCCESS;
10709	}
10710	break;
10711	}
10712
10713	default:
10714	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10715	rc = VINF_EM_RAW_EMULATE_INSTR;
10716	break;
10717	}
10718
10719	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10720	return rc;
10721	}
10722
10723
10724	/**
10725	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10726	* VM-exit.
10727	*/
10728	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10729	{
10730	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10731
10732	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10733	if (pVmxTransient->fWasGuestDebugStateActive)
10734	{
10735	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10736	HMVMX_RETURN_UNEXPECTED_EXIT();
10737	}
10738
10739	int rc = VERR_INTERNAL_ERROR_5;
10740	if ( !DBGFIsStepping(pVCpu)
10741	&& !pVCpu->hm.s.fSingleInstruction
10742	&& !pVmxTransient->fWasHyperDebugStateActive)
10743	{
10744	/* Don't intercept MOV DRx and #DB any more. */
10745	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10746	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10747	AssertRCReturn(rc, rc);
10748
10749	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10750	{
10751	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10752	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10753	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10754	AssertRCReturn(rc, rc);
10755	#endif
10756	}
10757
10758	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10759	VMMRZCallRing3Disable(pVCpu);
10760	HM_DISABLE_PREEMPT_IF_NEEDED();
10761
10762	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10763	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10764	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10765
10766	HM_RESTORE_PREEMPT_IF_NEEDED();
10767	VMMRZCallRing3Enable(pVCpu);
10768
10769	#ifdef VBOX_WITH_STATISTICS
10770	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10771	AssertRCReturn(rc, rc);
10772	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10773	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10774	else
10775	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10776	#endif
10777	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10778	return VINF_SUCCESS;
10779	}
10780
10781	/*
10782	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10783	* Update the segment registers and DR7 from the CPU.
10784	*/
10785	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10786	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10787	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10788	AssertRCReturn(rc, rc);
10789	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10790
10791	PVM pVM = pVCpu->CTX_SUFF(pVM);
10792	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10793	{
10794	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10795	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10796	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10797	if (RT_SUCCESS(rc))
10798	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10799	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10800	}
10801	else
10802	{
10803	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10804	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10805	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10806	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10807	}
10808
10809	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10810	if (RT_SUCCESS(rc))
10811	{
10812	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10813	AssertRCReturn(rc2, rc2);
10814	}
10815	return rc;
10816	}
10817
10818
10819	/**
10820	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10821	* Conditional VM-exit.
10822	*/
10823	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10824	{
10825	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10826	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10827
10828	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10829	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10830	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10831	return VINF_SUCCESS;
10832	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10833	return rc;
10834
10835	RTGCPHYS GCPhys = 0;
10836	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10837
10838	#if 0
10839	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10840	#else
10841	/* Aggressive state sync. for now. */
10842	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10843	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10844	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10845	#endif
10846	AssertRCReturn(rc, rc);
10847
10848	/*
10849	* If we succeed, resume guest execution.
10850	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10851	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10852	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10853	* weird case. See @bugref{6043}.
10854	*/
10855	PVM pVM = pVCpu->CTX_SUFF(pVM);
10856	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10857	rc = VBOXSTRICTRC_VAL(rc2);
10858	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10859	if ( rc == VINF_SUCCESS
10860	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10861	\|\| rc == VERR_PAGE_NOT_PRESENT)
10862	{
10863	/* Successfully handled MMIO operation. */
10864	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10865	\| HM_CHANGED_GUEST_RSP
10866	\| HM_CHANGED_GUEST_RFLAGS
10867	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10868	rc = VINF_SUCCESS;
10869	}
10870	return rc;
10871	}
10872
10873
10874	/**
10875	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10876	* VM-exit.
10877	*/
10878	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10879	{
10880	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10881	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10882
10883	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10884	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10885	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10886	return VINF_SUCCESS;
10887	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10888	return rc;
10889
10890	RTGCPHYS GCPhys = 0;
10891	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10892	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10893	#if 0
10894	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10895	#else
10896	/* Aggressive state sync. for now. */
10897	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10898	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10899	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10900	#endif
10901	AssertRCReturn(rc, rc);
10902
10903	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10904	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10905
10906	RTGCUINT uErrorCode = 0;
10907	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10908	uErrorCode \|= X86_TRAP_PF_ID;
10909	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10910	uErrorCode \|= X86_TRAP_PF_RW;
10911	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10912	uErrorCode \|= X86_TRAP_PF_P;
10913
10914	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10915
10916	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10917	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10918
10919	/* Handle the pagefault trap for the nested shadow table. */
10920	PVM pVM = pVCpu->CTX_SUFF(pVM);
10921	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10922	TRPMResetTrap(pVCpu);
10923
10924	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10925	if ( rc == VINF_SUCCESS
10926	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10927	\|\| rc == VERR_PAGE_NOT_PRESENT)
10928	{
10929	/* Successfully synced our nested page tables. */
10930	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10931	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10932	\| HM_CHANGED_GUEST_RSP
10933	\| HM_CHANGED_GUEST_RFLAGS);
10934	return VINF_SUCCESS;
10935	}
10936
10937	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10938	return rc;
10939	}
10940
10941	/** @} */
10942
10943	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10944	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10945	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10946
10947	/** @name VM-exit exception handlers.
10948	* @{
10949	*/
10950
10951	/**
10952	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10953	*/
10954	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10955	{
10956	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10958
10959	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10960	AssertRCReturn(rc, rc);
10961
10962	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10963	{
10964	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10965	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10966
10967	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10968	* provides VM-exit instruction length. If this causes problem later,
10969	* disassemble the instruction like it's done on AMD-V. */
10970	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10971	AssertRCReturn(rc2, rc2);
10972	return rc;
10973	}
10974
10975	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10976	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10977	return rc;
10978	}
10979
10980
10981	/**
10982	* VM-exit exception handler for #BP (Breakpoint exception).
10983	*/
10984	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10985	{
10986	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10987	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10988
10989	/** @todo Try optimize this by not saving the entire guest state unless
10990	* really needed. */
10991	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10992	AssertRCReturn(rc, rc);
10993
10994	PVM pVM = pVCpu->CTX_SUFF(pVM);
10995	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10996	if (rc == VINF_EM_RAW_GUEST_TRAP)
10997	{
10998	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10999	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11000	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11001	AssertRCReturn(rc, rc);
11002
11003	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11004	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11005	}
11006
11007	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
11008	return rc;
11009	}
11010
11011
11012	/**
11013	* VM-exit exception handler for #DB (Debug exception).
11014	*/
11015	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11016	{
11017	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11019	Log6(("XcptDB\n"));
11020
11021	/*
11022	* Get the DR6-like values from the exit qualification and pass it to DBGF
11023	* for processing.
11024	*/
11025	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11026	AssertRCReturn(rc, rc);
11027
11028	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11029	uint64_t uDR6 = X86_DR6_INIT_VAL;
11030	uDR6 \|= ( pVmxTransient->uExitQualification
11031	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
11032
11033	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11034	if (rc == VINF_EM_RAW_GUEST_TRAP)
11035	{
11036	/*
11037	* The exception was for the guest. Update DR6, DR7.GD and
11038	* IA32_DEBUGCTL.LBR before forwarding it.
11039	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11040	*/
11041	VMMRZCallRing3Disable(pVCpu);
11042	HM_DISABLE_PREEMPT_IF_NEEDED();
11043
11044	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11045	pMixedCtx->dr[6] \|= uDR6;
11046	if (CPUMIsGuestDebugStateActive(pVCpu))
11047	ASMSetDR6(pMixedCtx->dr[6]);
11048
11049	HM_RESTORE_PREEMPT_IF_NEEDED();
11050	VMMRZCallRing3Enable(pVCpu);
11051
11052	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11053	AssertRCReturn(rc, rc);
11054
11055	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11056	pMixedCtx->dr[7] &= ~X86_DR7_GD;
11057
11058	/* Paranoia. */
11059	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11060	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
11061
11062	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11063	AssertRCReturn(rc, rc);
11064
11065	/*
11066	* Raise #DB in the guest.
11067	*
11068	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11069	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11070	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11071	*
11072	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11073	*/
11074	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11075	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11076	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11077	AssertRCReturn(rc, rc);
11078	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11079	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11080	return VINF_SUCCESS;
11081	}
11082
11083	/*
11084	* Not a guest trap, must be a hypervisor related debug event then.
11085	* Update DR6 in case someone is interested in it.
11086	*/
11087	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11088	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11089	CPUMSetHyperDR6(pVCpu, uDR6);
11090
11091	return rc;
11092	}
11093
11094
11095	/**
11096	* VM-exit exception handler for #NM (Device-not-available exception: floating
11097	* point exception).
11098	*/
11099	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11100	{
11101	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11102
11103	/* We require CR0 and EFER. EFER is always up-to-date. */
11104	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11105	AssertRCReturn(rc, rc);
11106
11107	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11108	VMMRZCallRing3Disable(pVCpu);
11109	HM_DISABLE_PREEMPT_IF_NEEDED();
11110
11111	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11112	if (pVmxTransient->fWasGuestFPUStateActive)
11113	{
11114	rc = VINF_EM_RAW_GUEST_TRAP;
11115	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11116	}
11117	else
11118	{
11119	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11120	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11121	#endif
11122	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11123	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11124	}
11125
11126	HM_RESTORE_PREEMPT_IF_NEEDED();
11127	VMMRZCallRing3Enable(pVCpu);
11128
11129	if (rc == VINF_SUCCESS)
11130	{
11131	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11132	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11133	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11134	pVCpu->hm.s.fUseGuestFpu = true;
11135	}
11136	else
11137	{
11138	/* Forward #NM to the guest. */
11139	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11140	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11141	AssertRCReturn(rc, rc);
11142	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11143	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11144	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11145	}
11146
11147	return VINF_SUCCESS;
11148	}
11149
11150
11151	/**
11152	* VM-exit exception handler for #GP (General-protection exception).
11153	*
11154	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11155	*/
11156	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11157	{
11158	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11159	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11160
11161	int rc = VERR_INTERNAL_ERROR_5;
11162	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11163	{
11164	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11165	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11166	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11167	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11168	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11169	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11170	AssertRCReturn(rc, rc);
11171	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11172	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11173	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11174	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11175	return rc;
11176	#else
11177	/* We don't intercept #GP. */
11178	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11179	NOREF(pVmxTransient);
11180	return VERR_VMX_UNEXPECTED_EXCEPTION;
11181	#endif
11182	}
11183
11184	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11185	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11186
11187	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11188	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11189	AssertRCReturn(rc, rc);
11190
11191	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11192	uint32_t cbOp = 0;
11193	PVM pVM = pVCpu->CTX_SUFF(pVM);
11194	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11195	if (RT_SUCCESS(rc))
11196	{
11197	rc = VINF_SUCCESS;
11198	Assert(cbOp == pDis->cbInstr);
11199	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11200	switch (pDis->pCurInstr->uOpcode)
11201	{
11202	case OP_CLI:
11203	{
11204	pMixedCtx->eflags.Bits.u1IF = 0;
11205	pMixedCtx->rip += pDis->cbInstr;
11206	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11207	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11208	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11209	break;
11210	}
11211
11212	case OP_STI:
11213	{
11214	pMixedCtx->eflags.Bits.u1IF = 1;
11215	pMixedCtx->rip += pDis->cbInstr;
11216	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11217	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11218	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11219	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11220	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11221	break;
11222	}
11223
11224	case OP_HLT:
11225	{
11226	rc = VINF_EM_HALT;
11227	pMixedCtx->rip += pDis->cbInstr;
11228	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11229	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11230	break;
11231	}
11232
11233	case OP_POPF:
11234	{
11235	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11236	uint32_t cbParm;
11237	uint32_t uMask;
11238	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11239	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11240	{
11241	cbParm = 4;
11242	uMask = 0xffffffff;
11243	}
11244	else
11245	{
11246	cbParm = 2;
11247	uMask = 0xffff;
11248	}
11249
11250	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11251	RTGCPTR GCPtrStack = 0;
11252	X86EFLAGS Eflags;
11253	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11254	&GCPtrStack);
11255	if (RT_SUCCESS(rc))
11256	{
11257	Assert(sizeof(Eflags.u32) >= cbParm);
11258	Eflags.u32 = 0;
11259	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11260	}
11261	if (RT_FAILURE(rc))
11262	{
11263	rc = VERR_EM_INTERPRETER;
11264	break;
11265	}
11266	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11267	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11268	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11269	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11270	pMixedCtx->esp += cbParm;
11271	pMixedCtx->esp &= uMask;
11272	pMixedCtx->rip += pDis->cbInstr;
11273	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11274	\| HM_CHANGED_GUEST_RSP
11275	\| HM_CHANGED_GUEST_RFLAGS);
11276	/* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11277	if (fStepping)
11278	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11279
11280	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11281	break;
11282	}
11283
11284	case OP_PUSHF:
11285	{
11286	uint32_t cbParm;
11287	uint32_t uMask;
11288	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11289	{
11290	cbParm = 4;
11291	uMask = 0xffffffff;
11292	}
11293	else
11294	{
11295	cbParm = 2;
11296	uMask = 0xffff;
11297	}
11298
11299	/* Get the stack pointer & push the contents of eflags onto the stack. */
11300	RTGCPTR GCPtrStack = 0;
11301	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11302	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11303	if (RT_FAILURE(rc))
11304	{
11305	rc = VERR_EM_INTERPRETER;
11306	break;
11307	}
11308	X86EFLAGS Eflags = pMixedCtx->eflags;
11309	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11310	Eflags.Bits.u1RF = 0;
11311	Eflags.Bits.u1VM = 0;
11312
11313	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11314	if (RT_FAILURE(rc))
11315	{
11316	rc = VERR_EM_INTERPRETER;
11317	break;
11318	}
11319	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11320	pMixedCtx->esp -= cbParm;
11321	pMixedCtx->esp &= uMask;
11322	pMixedCtx->rip += pDis->cbInstr;
11323	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11324	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11325	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11326	break;
11327	}
11328
11329	case OP_IRET:
11330	{
11331	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11332	* instruction reference. */
11333	RTGCPTR GCPtrStack = 0;
11334	uint32_t uMask = 0xffff;
11335	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11336	uint16_t aIretFrame[3];
11337	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11338	{
11339	rc = VERR_EM_INTERPRETER;
11340	break;
11341	}
11342	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11343	&GCPtrStack);
11344	if (RT_SUCCESS(rc))
11345	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11346	if (RT_FAILURE(rc))
11347	{
11348	rc = VERR_EM_INTERPRETER;
11349	break;
11350	}
11351	pMixedCtx->eip = 0;
11352	pMixedCtx->ip = aIretFrame[0];
11353	pMixedCtx->cs.Sel = aIretFrame[1];
11354	pMixedCtx->cs.ValidSel = aIretFrame[1];
11355	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11356	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11357	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11358	pMixedCtx->sp += sizeof(aIretFrame);
11359	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11360	\| HM_CHANGED_GUEST_SEGMENT_REGS
11361	\| HM_CHANGED_GUEST_RSP
11362	\| HM_CHANGED_GUEST_RFLAGS);
11363	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11364	if (fStepping)
11365	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11366	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11367	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11368	break;
11369	}
11370
11371	case OP_INT:
11372	{
11373	uint16_t uVector = pDis->Param1.uValue & 0xff;
11374	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11375	/* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11377	break;
11378	}
11379
11380	case OP_INTO:
11381	{
11382	if (pMixedCtx->eflags.Bits.u1OF)
11383	{
11384	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11385	/* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11387	}
11388	break;
11389	}
11390
11391	default:
11392	{
11393	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11394	EMCODETYPE_SUPERVISOR);
11395	rc = VBOXSTRICTRC_VAL(rc2);
11396	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11397	/** @todo We have to set pending-debug exceptions here when the guest is
11398	* single-stepping depending on the instruction that was interpreted. */
11399	Log4(("#GP rc=%Rrc\n", rc));
11400	break;
11401	}
11402	}
11403	}
11404	else
11405	rc = VERR_EM_INTERPRETER;
11406
11407	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11408	("#GP Unexpected rc=%Rrc\n", rc));
11409	return rc;
11410	}
11411
11412
11413	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11414	/**
11415	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11416	* the exception reported in the VMX transient structure back into the VM.
11417	*
11418	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11419	* up-to-date.
11420	*/
11421	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11422	{
11423	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11424
11425	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11426	hmR0VmxCheckExitDueToEventDelivery(). */
11427	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11428	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11429	AssertRCReturn(rc, rc);
11430	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11431
11432	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11433	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11434	return VINF_SUCCESS;
11435	}
11436	#endif
11437
11438
11439	/**
11440	* VM-exit exception handler for #PF (Page-fault exception).
11441	*/
11442	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11443	{
11444	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11445	PVM pVM = pVCpu->CTX_SUFF(pVM);
11446	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11447	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11448	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11449	AssertRCReturn(rc, rc);
11450
11451	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11452	if (pVM->hm.s.fNestedPaging)
11453	{
11454	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11455	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11456	{
11457	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11458	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11459	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11460	}
11461	else
11462	{
11463	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11464	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11465	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11466	}
11467	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11468	return rc;
11469	}
11470	#else
11471	Assert(!pVM->hm.s.fNestedPaging);
11472	NOREF(pVM);
11473	#endif
11474
11475	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11476	AssertRCReturn(rc, rc);
11477
11478	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11479	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11480
11481	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11482	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11483	(RTGCPTR)pVmxTransient->uExitQualification);
11484
11485	Log4(("#PF: rc=%Rrc\n", rc));
11486	if (rc == VINF_SUCCESS)
11487	{
11488	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11489	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11490	* memory? We don't update the whole state here... */
11491	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11492	\| HM_CHANGED_GUEST_RSP
11493	\| HM_CHANGED_GUEST_RFLAGS
11494	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11495	TRPMResetTrap(pVCpu);
11496	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11497	return rc;
11498	}
11499	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11500	{
11501	if (!pVmxTransient->fVectoringPF)
11502	{
11503	/* It's a guest page fault and needs to be reflected to the guest. */
11504	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11505	TRPMResetTrap(pVCpu);
11506	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11507	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11508	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11509	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11510	}
11511	else
11512	{
11513	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11514	TRPMResetTrap(pVCpu);
11515	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11516	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11517	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11518	}
11519
11520	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11521	return VINF_SUCCESS;
11522	}
11523
11524	TRPMResetTrap(pVCpu);
11525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11526	return rc;
11527	}
11528
11529	/** @} */
11530

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 50614

Download in other formats: