HMVMXR0.cpp@ 49523

Last change on this file since 49523 was 49523, checked in by vboxsync, 11 years ago
VMM: Retire VBOX_WITH_AUTO_MSR_LOAD_RESTORE define.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 464.0 KB

Line
1	/* $Id: HMVMXR0.cpp 49523 2013-11-18 10:59:01Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	# define HMVMX_SAVE_FULL_GUEST_STATE
40	# define HMVMX_SYNC_FULL_GUEST_STATE
41	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	# define HMVMX_ALWAYS_TRAP_PF
44	# define HMVMX_ALWAYS_SWAP_FPU_STATE
45	# define HMVMX_ALWAYS_FLUSH_TLB
46	#endif
47
48
49	/*******************************************************************************
50	* Defined Constants And Macros *
51	*******************************************************************************/
52	#if defined(RT_ARCH_AMD64)
53	# define HMVMX_IS_64BIT_HOST_MODE() (true)
54	typedef RTHCUINTREG HMVMXHCUINTREG;
55	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56	extern "C" uint32_t g_fVMXIs64bitHost;
57	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58	typedef uint64_t HMVMXHCUINTREG;
59	#else
60	# define HMVMX_IS_64BIT_HOST_MODE() (false)
61	typedef RTHCUINTREG HMVMXHCUINTREG;
62	#endif
63
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/** Determine which tagged-TLB flush handler to use. */
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
70	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
71	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
72
73	/** @name Updated-guest-state flags.
74	* @{ */
75	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
76	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
77	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
78	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
79	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
80	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
81	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
82	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
83	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
84	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
85	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
86	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(16)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(17)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
108	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
109	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually (except #PF). #NM is also
142	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
143	* even in real-mode if we have Nested Paging support.
144	*/
145	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
146	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
147	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
148	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
149	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
150	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
151	\| RT_BIT(X86_XCPT_XF))
152
153	/**
154	* Exception bitmap mask for all contributory exceptions.
155	*
156	* Page fault is deliberately excluded here as it's conditional as to whether
157	* it's contributory or benign. Page faults are handled separately.
158	*/
159	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
160	\| RT_BIT(X86_XCPT_DE))
161
162	/** Maximum VM-instruction error number. */
163	#define HMVMX_INSTR_ERROR_MAX 28
164
165	/** Profiling macro. */
166	#ifdef HM_PROFILE_EXIT_DISPATCH
167	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
168	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
169	#else
170	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
172	#endif
173
174	/** Assert that preemption is disabled or covered by thread-context hooks. */
175	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
176	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
177
178	/** Assert that we haven't migrated CPUs when thread-context hooks are not
179	* used. */
180	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
181	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
182	("Illegal migration! Entered on CPU %u Current %u\n", \
183	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
184
185	/** Helper macro for VM-exit handlers called unexpectedly. */
186	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
187	do { \
188	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
189	return VERR_VMX_UNEXPECTED_EXIT; \
190	} while (0)
191
192
193	/*******************************************************************************
194	* Structures and Typedefs *
195	*******************************************************************************/
196	/**
197	* VMX transient state.
198	*
199	* A state structure for holding miscellaneous information across
200	* VMX non-root operation and restored after the transition.
201	*/
202	typedef struct VMXTRANSIENT
203	{
204	/** The host's rflags/eflags. */
205	RTCCUINTREG uEflags;
206	#if HC_ARCH_BITS == 32
207	uint32_t u32Alignment0;
208	#endif
209	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
210	uint64_t u64LStarMsr;
211	/** The guest's TPR value used for TPR shadowing. */
212	uint8_t u8GuestTpr;
213	/** Alignment. */
214	uint8_t abAlignment0[7];
215
216	/** The basic VM-exit reason. */
217	uint16_t uExitReason;
218	/** Alignment. */
219	uint16_t u16Alignment0;
220	/** The VM-exit interruption error code. */
221	uint32_t uExitIntErrorCode;
222	/** The VM-exit exit qualification. */
223	uint64_t uExitQualification;
224
225	/** The VM-exit interruption-information field. */
226	uint32_t uExitIntInfo;
227	/** The VM-exit instruction-length field. */
228	uint32_t cbInstr;
229	/** The VM-exit instruction-information field. */
230	union
231	{
232	/** Plain unsigned int representation. */
233	uint32_t u;
234	/** INS and OUTS information. */
235	struct
236	{
237	uint32_t u6Reserved0 : 7;
238	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239	uint32_t u3AddrSize : 3;
240	uint32_t u5Reserved1 : 5;
241	/** The segment register (X86_SREG_XXX). */
242	uint32_t iSegReg : 3;
243	uint32_t uReserved2 : 14;
244	} StrIo;
245	} ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Alignment. */
249	uint8_t abAlignment1[3];
250
251	/** The VM-entry interruption-information field. */
252	uint32_t uEntryIntInfo;
253	/** The VM-entry exception error code field. */
254	uint32_t uEntryXcptErrorCode;
255	/** The VM-entry instruction length field. */
256	uint32_t cbEntryInstr;
257
258	/** IDT-vectoring information field. */
259	uint32_t uIdtVectoringInfo;
260	/** IDT-vectoring error code. */
261	uint32_t uIdtVectoringErrorCode;
262
263	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
264	uint32_t fVmcsFieldsRead;
265
266	/** Whether the guest FPU was active at the time of VM-exit. */
267	bool fWasGuestFPUStateActive;
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting should be setup before VM-entry. */
273	bool fUpdateTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringPF;
277	} VMXTRANSIENT;
278	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
282	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
283	/** Pointer to VMX transient state. */
284	typedef VMXTRANSIENT *PVMXTRANSIENT;
285
286
287	/**
288	* MSR-bitmap read permissions.
289	*/
290	typedef enum VMXMSREXITREAD
291	{
292	/** Reading this MSR causes a VM-exit. */
293	VMXMSREXIT_INTERCEPT_READ = 0xb,
294	/** Reading this MSR does not cause a VM-exit. */
295	VMXMSREXIT_PASSTHRU_READ
296	} VMXMSREXITREAD;
297	/** Pointer to MSR-bitmap read permissions. */
298	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
299
300	/**
301	* MSR-bitmap write permissions.
302	*/
303	typedef enum VMXMSREXITWRITE
304	{
305	/** Writing to this MSR causes a VM-exit. */
306	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
307	/** Writing to this MSR does not cause a VM-exit. */
308	VMXMSREXIT_PASSTHRU_WRITE
309	} VMXMSREXITWRITE;
310	/** Pointer to MSR-bitmap write permissions. */
311	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
312
313
314	/**
315	* VMX VM-exit handler.
316	*
317	* @returns VBox status code.
318	* @param pVCpu Pointer to the VMCPU.
319	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
320	* out-of-sync. Make sure to update the required
321	* fields before using them.
322	* @param pVmxTransient Pointer to the VMX-transient structure.
323	*/
324	#ifndef HMVMX_USE_FUNCTION_TABLE
325	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
326	#else
327	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
328	/** Pointer to VM-exit handler. */
329	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
330	#endif
331
332
333	/*******************************************************************************
334	* Internal Functions *
335	*******************************************************************************/
336	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
337	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
338	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
339	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
340	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
341	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
342	#endif
343	#ifndef HMVMX_USE_FUNCTION_TABLE
344	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
345	# define HMVMX_EXIT_DECL static int
346	#else
347	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
348	#endif
349
350	/** @name VM-exit handlers.
351	* @{
352	*/
353	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
354	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
355	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
356	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
357	static FNVMXEXITHANDLER hmR0VmxExitSipi;
358	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
359	static FNVMXEXITHANDLER hmR0VmxExitSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
361	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
363	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
364	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
365	static FNVMXEXITHANDLER hmR0VmxExitHlt;
366	static FNVMXEXITHANDLER hmR0VmxExitInvd;
367	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
368	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
369	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
370	static FNVMXEXITHANDLER hmR0VmxExitRsm;
371	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
372	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
373	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
374	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
375	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
376	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
378	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
379	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
380	static FNVMXEXITHANDLER hmR0VmxExitMwait;
381	static FNVMXEXITHANDLER hmR0VmxExitMtf;
382	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
383	static FNVMXEXITHANDLER hmR0VmxExitPause;
384	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
385	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
386	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
387	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
390	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
391	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
392	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
393	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
394	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
395	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
396	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
397	/** @} */
398
399	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
400	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
406	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407	#endif
408	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
409
410	/*******************************************************************************
411	* Global Variables *
412	*******************************************************************************/
413	#ifdef HMVMX_USE_FUNCTION_TABLE
414
415	/**
416	* VMX_EXIT dispatch table.
417	*/
418	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
419	{
420	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
421	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
422	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
423	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
424	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
425	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
426	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
427	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
428	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
429	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
430	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
431	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
432	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
433	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
434	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
435	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
436	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
437	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
438	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
439	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
440	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
441	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
442	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
443	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
444	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
445	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
446	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
447	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
448	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
449	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
450	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
451	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
452	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
453	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
454	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
455	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
456	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
457	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
458	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
459	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
460	/* 40 UNDEFINED */ hmR0VmxExitPause,
461	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
462	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
463	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
464	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
465	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
466	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
467	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
469	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
470	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
471	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
472	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
473	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
474	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
475	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
476	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
477	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
478	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
479	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
480	};
481	#endif /* HMVMX_USE_FUNCTION_TABLE */
482
483	#ifdef VBOX_STRICT
484	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
485	{
486	/* 0 */ "(Not Used)",
487	/* 1 */ "VMCALL executed in VMX root operation.",
488	/* 2 */ "VMCLEAR with invalid physical address.",
489	/* 3 */ "VMCLEAR with VMXON pointer.",
490	/* 4 */ "VMLAUNCH with non-clear VMCS.",
491	/* 5 */ "VMRESUME with non-launched VMCS.",
492	/* 6 */ "VMRESUME after VMXOFF",
493	/* 7 */ "VM entry with invalid control fields.",
494	/* 8 */ "VM entry with invalid host state fields.",
495	/* 9 */ "VMPTRLD with invalid physical address.",
496	/* 10 */ "VMPTRLD with VMXON pointer.",
497	/* 11 */ "VMPTRLD with incorrect revision identifier.",
498	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
499	/* 13 */ "VMWRITE to read-only VMCS component.",
500	/* 14 */ "(Not Used)",
501	/* 15 */ "VMXON executed in VMX root operation.",
502	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
503	/* 17 */ "VM entry with non-launched executing VMCS.",
504	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
505	/* 19 */ "VMCALL with non-clear VMCS.",
506	/* 20 */ "VMCALL with invalid VM-exit control fields.",
507	/* 21 */ "(Not Used)",
508	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
509	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
510	/* 24 */ "VMCALL with invalid SMM-monitor features.",
511	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
512	/* 26 */ "VM entry with events blocked by MOV SS.",
513	/* 27 */ "(Not Used)",
514	/* 28 */ "Invalid operand to INVEPT/INVVPID."
515	};
516	#endif /* VBOX_STRICT */
517
518
519
520	/**
521	* Updates the VM's last error record. If there was a VMX instruction error,
522	* reads the error data from the VMCS and updates VCPU's last error record as
523	* well.
524	*
525	* @param pVM Pointer to the VM.
526	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
527	* VERR_VMX_UNABLE_TO_START_VM or
528	* VERR_VMX_INVALID_VMCS_FIELD).
529	* @param rc The error code.
530	*/
531	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
532	{
533	AssertPtr(pVM);
534	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
535	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
536	{
537	AssertPtrReturnVoid(pVCpu);
538	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
539	}
540	pVM->hm.s.lLastError = rc;
541	}
542
543
544	/**
545	* Reads the VM-entry interruption-information field from the VMCS into the VMX
546	* transient structure.
547	*
548	* @returns VBox status code.
549	* @param pVmxTransient Pointer to the VMX transient structure.
550	*
551	* @remarks No-long-jump zone!!!
552	*/
553	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
554	{
555	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
556	AssertRCReturn(rc, rc);
557	return VINF_SUCCESS;
558	}
559
560
561	/**
562	* Reads the VM-entry exception error code field from the VMCS into
563	* the VMX transient structure.
564	*
565	* @returns VBox status code.
566	* @param pVmxTransient Pointer to the VMX transient structure.
567	*
568	* @remarks No-long-jump zone!!!
569	*/
570	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
571	{
572	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
573	AssertRCReturn(rc, rc);
574	return VINF_SUCCESS;
575	}
576
577
578	/**
579	* Reads the VM-entry exception error code field from the VMCS into
580	* the VMX transient structure.
581	*
582	* @returns VBox status code.
583	* @param pVmxTransient Pointer to the VMX transient structure.
584	*
585	* @remarks No-long-jump zone!!!
586	*/
587	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
588	{
589	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
590	AssertRCReturn(rc, rc);
591	return VINF_SUCCESS;
592	}
593
594
595	/**
596	* Reads the VM-exit interruption-information field from the VMCS into the VMX
597	* transient structure.
598	*
599	* @returns VBox status code.
600	* @param pVmxTransient Pointer to the VMX transient structure.
601	*/
602	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
603	{
604	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
605	{
606	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
607	AssertRCReturn(rc, rc);
608	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
609	}
610	return VINF_SUCCESS;
611	}
612
613
614	/**
615	* Reads the VM-exit interruption error code from the VMCS into the VMX
616	* transient structure.
617	*
618	* @returns VBox status code.
619	* @param pVmxTransient Pointer to the VMX transient structure.
620	*/
621	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
622	{
623	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
624	{
625	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
626	AssertRCReturn(rc, rc);
627	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
628	}
629	return VINF_SUCCESS;
630	}
631
632
633	/**
634	* Reads the VM-exit instruction length field from the VMCS into the VMX
635	* transient structure.
636	*
637	* @returns VBox status code.
638	* @param pVCpu Pointer to the VMCPU.
639	* @param pVmxTransient Pointer to the VMX transient structure.
640	*/
641	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
642	{
643	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
644	{
645	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
646	AssertRCReturn(rc, rc);
647	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
648	}
649	return VINF_SUCCESS;
650	}
651
652
653	/**
654	* Reads the VM-exit instruction-information field from the VMCS into
655	* the VMX transient structure.
656	*
657	* @returns VBox status code.
658	* @param pVmxTransient Pointer to the VMX transient structure.
659	*/
660	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
661	{
662	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
665	AssertRCReturn(rc, rc);
666	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
667	}
668	return VINF_SUCCESS;
669	}
670
671
672	/**
673	* Reads the exit qualification from the VMCS into the VMX transient structure.
674	*
675	* @returns VBox status code.
676	* @param pVmxTransient Pointer to the VMX transient structure.
677	*/
678	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMXTRANSIENT pVmxTransient)
679	{
680	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
681	{
682	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
683	AssertRCReturn(rc, rc);
684	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
685	}
686	return VINF_SUCCESS;
687	}
688
689
690	/**
691	* Reads the IDT-vectoring information field from the VMCS into the VMX
692	* transient structure.
693	*
694	* @returns VBox status code.
695	* @param pVmxTransient Pointer to the VMX transient structure.
696	*
697	* @remarks No-long-jump zone!!!
698	*/
699	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
700	{
701	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
702	{
703	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
704	AssertRCReturn(rc, rc);
705	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
706	}
707	return VINF_SUCCESS;
708	}
709
710
711	/**
712	* Reads the IDT-vectoring error code from the VMCS into the VMX
713	* transient structure.
714	*
715	* @returns VBox status code.
716	* @param pVmxTransient Pointer to the VMX transient structure.
717	*/
718	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
719	{
720	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
721	{
722	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
723	AssertRCReturn(rc, rc);
724	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
725	}
726	return VINF_SUCCESS;
727	}
728
729
730	/**
731	* Enters VMX root mode operation on the current CPU.
732	*
733	* @returns VBox status code.
734	* @param pVM Pointer to the VM (optional, can be NULL, after
735	* a resume).
736	* @param HCPhysCpuPage Physical address of the VMXON region.
737	* @param pvCpuPage Pointer to the VMXON region.
738	*/
739	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
740	{
741	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
742	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
743	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
744
745	if (pVM)
746	{
747	/* Write the VMCS revision dword to the VMXON region. */
748	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
749	}
750
751	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
752	RTCCUINTREG uEflags = ASMIntDisableFlags();
753
754	/* Enable the VMX bit in CR4 if necessary. */
755	RTCCUINTREG uCr4 = ASMGetCR4();
756	if (!(uCr4 & X86_CR4_VMXE))
757	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
758
759	/* Enter VMX root mode. */
760	int rc = VMXEnable(HCPhysCpuPage);
761	if (RT_FAILURE(rc))
762	ASMSetCR4(uCr4);
763
764	/* Restore interrupts. */
765	ASMSetFlags(uEflags);
766	return rc;
767	}
768
769
770	/**
771	* Exits VMX root mode operation on the current CPU.
772	*
773	* @returns VBox status code.
774	*/
775	static int hmR0VmxLeaveRootMode(void)
776	{
777	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
778
779	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
780	RTCCUINTREG uEflags = ASMIntDisableFlags();
781
782	/* If we're for some reason not in VMX root mode, then don't leave it. */
783	RTCCUINTREG uHostCR4 = ASMGetCR4();
784
785	int rc;
786	if (uHostCR4 & X86_CR4_VMXE)
787	{
788	/* Exit VMX root mode and clear the VMX bit in CR4. */
789	VMXDisable();
790	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
791	rc = VINF_SUCCESS;
792	}
793	else
794	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
795
796	/* Restore interrupts. */
797	ASMSetFlags(uEflags);
798	return rc;
799	}
800
801
802	/**
803	* Allocates and maps one physically contiguous page. The allocated page is
804	* zero'd out. (Used by various VT-x structures).
805	*
806	* @returns IPRT status code.
807	* @param pMemObj Pointer to the ring-0 memory object.
808	* @param ppVirt Where to store the virtual address of the
809	* allocation.
810	* @param pPhys Where to store the physical address of the
811	* allocation.
812	*/
813	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
814	{
815	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
816	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
817	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
818
819	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
820	if (RT_FAILURE(rc))
821	return rc;
822	ppVirt = RTR0MemObjAddress(pMemObj);
823	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
824	ASMMemZero32(*ppVirt, PAGE_SIZE);
825	return VINF_SUCCESS;
826	}
827
828
829	/**
830	* Frees and unmaps an allocated physical page.
831	*
832	* @param pMemObj Pointer to the ring-0 memory object.
833	* @param ppVirt Where to re-initialize the virtual address of
834	* allocation as 0.
835	* @param pHCPhys Where to re-initialize the physical address of the
836	* allocation as 0.
837	*/
838	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
839	{
840	AssertPtr(pMemObj);
841	AssertPtr(ppVirt);
842	AssertPtr(pHCPhys);
843	if (*pMemObj != NIL_RTR0MEMOBJ)
844	{
845	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
846	AssertRC(rc);
847	*pMemObj = NIL_RTR0MEMOBJ;
848	*ppVirt = 0;
849	*pHCPhys = 0;
850	}
851	}
852
853
854	/**
855	* Worker function to free VT-x related structures.
856	*
857	* @returns IPRT status code.
858	* @param pVM Pointer to the VM.
859	*/
860	static void hmR0VmxStructsFree(PVM pVM)
861	{
862	for (VMCPUID i = 0; i < pVM->cCpus; i++)
863	{
864	PVMCPU pVCpu = &pVM->aCpus[i];
865	AssertPtr(pVCpu);
866
867	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
868	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
869
870	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
872
873	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
875	}
876
877	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
878	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
879	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
880	#endif
881	}
882
883
884	/**
885	* Worker function to allocate VT-x related VM structures.
886	*
887	* @returns IPRT status code.
888	* @param pVM Pointer to the VM.
889	*/
890	static int hmR0VmxStructsAlloc(PVM pVM)
891	{
892	/*
893	* Initialize members up-front so we can cleanup properly on allocation failure.
894	*/
895	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
896	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
897	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
898	pVM->hm.s.vmx.HCPhys##a_Name = 0;
899
900	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
901	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
902	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
903	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
904
905	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
906	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
907	#endif
908	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
909
910	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
911	for (VMCPUID i = 0; i < pVM->cCpus; i++)
912	{
913	PVMCPU pVCpu = &pVM->aCpus[i];
914	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
915	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
916	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
919	}
920	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
921	#undef VMXLOCAL_INIT_VM_MEMOBJ
922
923	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
924	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
925	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
926	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
927
928	/*
929	* Allocate all the VT-x structures.
930	*/
931	int rc = VINF_SUCCESS;
932	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
933	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
934	if (RT_FAILURE(rc))
935	goto cleanup;
936	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
937	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
938	#endif
939
940	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
941	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
942	{
943	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
944	&pVM->hm.s.vmx.HCPhysApicAccess);
945	if (RT_FAILURE(rc))
946	goto cleanup;
947	}
948
949	/*
950	* Initialize per-VCPU VT-x structures.
951	*/
952	for (VMCPUID i = 0; i < pVM->cCpus; i++)
953	{
954	PVMCPU pVCpu = &pVM->aCpus[i];
955	AssertPtr(pVCpu);
956
957	/* Allocate the VM control structure (VMCS). */
958	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
959	if (RT_FAILURE(rc))
960	goto cleanup;
961
962	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
963	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
964	{
965	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
966	&pVCpu->hm.s.vmx.HCPhysVirtApic);
967	if (RT_FAILURE(rc))
968	goto cleanup;
969	}
970
971	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
972	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
973	{
974	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
975	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
976	if (RT_FAILURE(rc))
977	goto cleanup;
978	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
979	}
980
981	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
982	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
983	if (RT_FAILURE(rc))
984	goto cleanup;
985
986	/* Allocate the VM-exit MSR-load page for the host MSRs. */
987	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
988	if (RT_FAILURE(rc))
989	goto cleanup;
990	}
991
992	return VINF_SUCCESS;
993
994	cleanup:
995	hmR0VmxStructsFree(pVM);
996	return rc;
997	}
998
999
1000	/**
1001	* Does global VT-x initialization (called during module initialization).
1002	*
1003	* @returns VBox status code.
1004	*/
1005	VMMR0DECL(int) VMXR0GlobalInit(void)
1006	{
1007	#ifdef HMVMX_USE_FUNCTION_TABLE
1008	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1009	# ifdef VBOX_STRICT
1010	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1011	Assert(g_apfnVMExitHandlers[i]);
1012	# endif
1013	#endif
1014	return VINF_SUCCESS;
1015	}
1016
1017
1018	/**
1019	* Does global VT-x termination (called during module termination).
1020	*/
1021	VMMR0DECL(void) VMXR0GlobalTerm()
1022	{
1023	/* Nothing to do currently. */
1024	}
1025
1026
1027	/**
1028	* Sets up and activates VT-x on the current CPU.
1029	*
1030	* @returns VBox status code.
1031	* @param pCpu Pointer to the global CPU info struct.
1032	* @param pVM Pointer to the VM (can be NULL after a host resume
1033	* operation).
1034	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1035	* fEnabledByHost is true).
1036	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1037	* @a fEnabledByHost is true).
1038	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1039	* enable VT-x on the host.
1040	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1041	*/
1042	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1043	void *pvMsrs)
1044	{
1045	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1046	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1047	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1048
1049	/* Enable VT-x if it's not already enabled by the host. */
1050	if (!fEnabledByHost)
1051	{
1052	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1053	if (RT_FAILURE(rc))
1054	return rc;
1055	}
1056
1057	/*
1058	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1059	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1060	*/
1061	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1062	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1063	{
1064	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1065	pCpu->fFlushAsidBeforeUse = false;
1066	}
1067	else
1068	pCpu->fFlushAsidBeforeUse = true;
1069
1070	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1071	++pCpu->cTlbFlushes;
1072
1073	return VINF_SUCCESS;
1074	}
1075
1076
1077	/**
1078	* Deactivates VT-x on the current CPU.
1079	*
1080	* @returns VBox status code.
1081	* @param pCpu Pointer to the global CPU info struct.
1082	* @param pvCpuPage Pointer to the VMXON region.
1083	* @param HCPhysCpuPage Physical address of the VMXON region.
1084	*
1085	* @remarks This function should never be called when SUPR0EnableVTx() or
1086	* similar was used to enable VT-x on the host.
1087	*/
1088	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1089	{
1090	NOREF(pCpu);
1091	NOREF(pvCpuPage);
1092	NOREF(HCPhysCpuPage);
1093
1094	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1095	return hmR0VmxLeaveRootMode();
1096	}
1097
1098
1099	/**
1100	* Sets the permission bits for the specified MSR in the MSR bitmap.
1101	*
1102	* @param pVCpu Pointer to the VMCPU.
1103	* @param uMSR The MSR value.
1104	* @param enmRead Whether reading this MSR causes a VM-exit.
1105	* @param enmWrite Whether writing this MSR causes a VM-exit.
1106	*/
1107	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1108	{
1109	int32_t iBit;
1110	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1111
1112	/*
1113	* Layout:
1114	* 0x000 - 0x3ff - Low MSR read bits
1115	* 0x400 - 0x7ff - High MSR read bits
1116	* 0x800 - 0xbff - Low MSR write bits
1117	* 0xc00 - 0xfff - High MSR write bits
1118	*/
1119	if (uMsr <= 0x00001FFF)
1120	iBit = uMsr;
1121	else if ( uMsr >= 0xC0000000
1122	&& uMsr <= 0xC0001FFF)
1123	{
1124	iBit = (uMsr - 0xC0000000);
1125	pbMsrBitmap += 0x400;
1126	}
1127	else
1128	{
1129	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1130	return;
1131	}
1132
1133	Assert(iBit <= 0x1fff);
1134	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1135	ASMBitSet(pbMsrBitmap, iBit);
1136	else
1137	ASMBitClear(pbMsrBitmap, iBit);
1138
1139	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1140	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1141	else
1142	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1143	}
1144
1145
1146	#ifdef VBOX_STRICT
1147	/**
1148	* Gets the permission bits for the specified MSR in the MSR bitmap.
1149	*
1150	* @returns VBox status code.
1151	* @retval VINF_SUCCESS if the specified MSR is found.
1152	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1153	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1154	*
1155	* @param pVCpu Pointer to the VMCPU.
1156	* @param uMsr The MSR.
1157	* @param penmRead Where to store the read permissions.
1158	* @param penmWrite Where to store the write permissions.
1159	*/
1160	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1161	{
1162	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1163	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1164	int32_t iBit;
1165	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1166
1167	/* See hmR0VmxSetMsrPermission() for the layout. */
1168	if (uMsr <= 0x00001FFF)
1169	iBit = uMsr;
1170	else if ( uMsr >= 0xC0000000
1171	&& uMsr <= 0xC0001FFF)
1172	{
1173	iBit = (uMsr - 0xC0000000);
1174	pbMsrBitmap += 0x400;
1175	}
1176	else
1177	{
1178	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1179	return VERR_NOT_SUPPORTED;
1180	}
1181
1182	Assert(iBit <= 0x1fff);
1183	if (ASMBitTest(pbMsrBitmap, iBit))
1184	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1185	else
1186	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1187
1188	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1189	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1190	else
1191	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1192	return VINF_SUCCESS;
1193	}
1194	#endif /* VBOX_STRICT */
1195
1196
1197	/**
1198	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1199	* area.
1200	*
1201	* @returns VBox status code.
1202	* @param pVCpu Pointer to the VMCPU.
1203	* @param cMsrs The number of MSRs.
1204	*/
1205	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1206	{
1207	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1208	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1209	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1210	{
1211	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1212	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1213	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1214	}
1215
1216	/* Update number of guest MSRs to load/store across the world-switch. */
1217	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1218	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1219
1220	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update the VCPU's copy of the MSR count. */
1224	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1225
1226	return VINF_SUCCESS;
1227	}
1228
1229
1230	/**
1231	* Adds a new (or updates the value of an existing) guest/host MSR
1232	* pair to be swapped during the world-switch as part of the
1233	* auto-load/store MSR area in the VMCS.
1234	*
1235	* @returns VBox status code.
1236	* @param pVCpu Pointer to the VMCPU.
1237	* @param uMsr The MSR.
1238	* @param uGuestMsr Value of the guest MSR.
1239	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1240	* necessary.
1241	*/
1242	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1243	{
1244	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1245	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1246	uint32_t i;
1247	for (i = 0; i < cMsrs; i++)
1248	{
1249	if (pGuestMsr->u32Msr == uMsr)
1250	break;
1251	pGuestMsr++;
1252	}
1253
1254	bool fAdded = false;
1255	if (i == cMsrs)
1256	{
1257	++cMsrs;
1258	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1259	AssertRCReturn(rc, rc);
1260
1261	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1262	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1263	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1264
1265	fAdded = true;
1266	}
1267
1268	/* Update the MSR values in the auto-load/store MSR area. */
1269	pGuestMsr->u32Msr = uMsr;
1270	pGuestMsr->u64Value = uGuestMsrValue;
1271
1272	/* Create/update the MSR slot in the host MSR area. */
1273	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1274	pHostMsr += i;
1275	pHostMsr->u32Msr = uMsr;
1276
1277	/*
1278	* Update the host MSR only when requested by the called AND when we're
1279	* adding it to the auto-load/store area. Otherwise, it would have been
1280	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1281	*/
1282	if ( fAdded
1283	&& fUpdateHostMsr)
1284	{
1285	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1286	}
1287
1288	return VINF_SUCCESS;
1289	}
1290
1291
1292	/**
1293	* Removes a guest/shost MSR pair to be swapped during the world-switch from the
1294	* auto-load/store MSR area in the VMCS.
1295	*
1296	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1297	* area.
1298	*
1299	* @returns VBox status code.
1300	* @param pVCpu Pointer to the VMCPU.
1301	* @param uMsr The MSR.
1302	*/
1303	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1304	{
1305	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1306	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1307	for (uint32_t i = 0; i < cMsrs; i++)
1308	{
1309	/* Find the MSR. */
1310	if (pGuestMsr->u32Msr == uMsr)
1311	{
1312	/* If it's the last MSR, simply reduce the count. */
1313	if (i == cMsrs - 1)
1314	{
1315	--cMsrs;
1316	break;
1317	}
1318
1319	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1320	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1321	pLastGuestMsr += cMsrs;
1322	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1323	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1324
1325	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1326	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1327	pLastHostMsr += cMsrs;
1328	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1329	pHostMsr->u64Value = pLastHostMsr->u64Value;
1330	--cMsrs;
1331	break;
1332	}
1333	pGuestMsr++;
1334	}
1335
1336	/* Update the VMCS if the count changed (meaning the MSR was found). */
1337	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1338	{
1339	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1340	AssertRCReturn(rc, rc);
1341
1342	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1343	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1344	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1345	}
1346
1347	return VINF_SUCCESS;
1348	}
1349
1350
1351	/**
1352	* Checks if the specified guest MSR is part of the auto-load/store area in
1353	* the VMCS.
1354	*
1355	* @returns true if found, false otherwise.
1356	* @param pVCpu Pointer to the VMCPU.
1357	* @param uMsr The MSR to find.
1358	*/
1359	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1360	{
1361	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1362	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1363
1364	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1365	{
1366	if (pGuestMsr->u32Msr == uMsr)
1367	return true;
1368	}
1369	return false;
1370	}
1371
1372
1373	/**
1374	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1375	*
1376	* @param pVCpu Pointer to the VMCPU.
1377	*/
1378	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1379	{
1380	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1381	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1382	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384
1385	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1386	{
1387	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1388	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1389	}
1390
1391	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1392	}
1393
1394
1395	#ifdef VBOX_STRICT
1396	/**
1397	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1398	* VMCS are correct.
1399	*
1400	* @param pVCpu Pointer to the VMCPU.
1401	*/
1402	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1403	{
1404	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1405
1406	/* Verify MSR counts in the VMCS are what we think it should be. */
1407	uint32_t cMsrs;
1408	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1409	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1410
1411	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1412	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1413
1414	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1415	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1416
1417	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1418	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1419	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1420	{
1421	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1422	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1423	pGuestMsr->u32Msr));
1424
1425	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1426	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1427	pHostMsr->u64Value, u64Msr));
1428
1429	/* Verify that the permissions are as expected in the MSR bitmap. */
1430	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1431	{
1432	VMXMSREXITREAD enmRead;
1433	VMXMSREXITWRITE enmWrite;
1434	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1435	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1436	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1437	pGuestMsr->u32Msr));
1438	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1439	pGuestMsr->u32Msr));
1440	}
1441	}
1442	}
1443	# endif /* VBOX_STRICT */
1444
1445
1446	/**
1447	* Flushes the TLB using EPT.
1448	*
1449	* @returns VBox status code.
1450	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1451	* enmFlush).
1452	* @param enmFlush Type of flush.
1453	*
1454	* @remarks Caller is responsible for making sure this function is called only
1455	* when NestedPaging is supported and providing @a enmFlush that is
1456	* supported by the CPU.
1457	* @remarks Can be called with interrupts disabled.
1458	*/
1459	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1460	{
1461	uint64_t au64Descriptor[2];
1462	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1463	au64Descriptor[0] = 0;
1464	else
1465	{
1466	Assert(pVCpu);
1467	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1468	}
1469	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1470
1471	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1472	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1473	rc));
1474	if ( RT_SUCCESS(rc)
1475	&& pVCpu)
1476	{
1477	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1478	}
1479	}
1480
1481
1482	/**
1483	* Flushes the TLB using VPID.
1484	*
1485	* @returns VBox status code.
1486	* @param pVM Pointer to the VM.
1487	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1488	* enmFlush).
1489	* @param enmFlush Type of flush.
1490	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1491	* on @a enmFlush).
1492	*
1493	* @remarks Can be called with interrupts disabled.
1494	*/
1495	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1496	{
1497	NOREF(pVM);
1498	AssertPtr(pVM);
1499	Assert(pVM->hm.s.vmx.fVpid);
1500
1501	uint64_t au64Descriptor[2];
1502	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1503	{
1504	au64Descriptor[0] = 0;
1505	au64Descriptor[1] = 0;
1506	}
1507	else
1508	{
1509	AssertPtr(pVCpu);
1510	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1511	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1512	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1513	au64Descriptor[1] = GCPtr;
1514	}
1515
1516	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1517	AssertMsg(rc == VINF_SUCCESS,
1518	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1519	if ( RT_SUCCESS(rc)
1520	&& pVCpu)
1521	{
1522	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1523	}
1524	}
1525
1526
1527	/**
1528	* Invalidates a guest page by guest virtual address. Only relevant for
1529	* EPT/VPID, otherwise there is nothing really to invalidate.
1530	*
1531	* @returns VBox status code.
1532	* @param pVM Pointer to the VM.
1533	* @param pVCpu Pointer to the VMCPU.
1534	* @param GCVirt Guest virtual address of the page to invalidate.
1535	*/
1536	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1537	{
1538	AssertPtr(pVM);
1539	AssertPtr(pVCpu);
1540	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1541
1542	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1543	if (!fFlushPending)
1544	{
1545	/*
1546	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1547	* See @bugref{6043} and @bugref{6177}.
1548	*
1549	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1550	* function maybe called in a loop with individual addresses.
1551	*/
1552	if (pVM->hm.s.vmx.fVpid)
1553	{
1554	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1555	{
1556	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1557	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1558	}
1559	else
1560	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1561	}
1562	else if (pVM->hm.s.fNestedPaging)
1563	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1564	}
1565
1566	return VINF_SUCCESS;
1567	}
1568
1569
1570	/**
1571	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1572	* otherwise there is nothing really to invalidate.
1573	*
1574	* @returns VBox status code.
1575	* @param pVM Pointer to the VM.
1576	* @param pVCpu Pointer to the VMCPU.
1577	* @param GCPhys Guest physical address of the page to invalidate.
1578	*/
1579	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1580	{
1581	NOREF(pVM); NOREF(GCPhys);
1582	LogFlowFunc(("%RGp\n", GCPhys));
1583
1584	/*
1585	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1586	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1587	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1588	*/
1589	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1590	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1591	return VINF_SUCCESS;
1592	}
1593
1594
1595	/**
1596	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1597	* case where neither EPT nor VPID is supported by the CPU.
1598	*
1599	* @param pVM Pointer to the VM.
1600	* @param pVCpu Pointer to the VMCPU.
1601	* @param pCpu Pointer to the global HM struct.
1602	*
1603	* @remarks Called with interrupts disabled.
1604	*/
1605	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1606	{
1607	AssertPtr(pVCpu);
1608	AssertPtr(pCpu);
1609	NOREF(pVM);
1610
1611	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1612
1613	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1614	#if 0
1615	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1616	pVCpu->hm.s.TlbShootdown.cPages = 0;
1617	#endif
1618
1619	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1620	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1621	pVCpu->hm.s.fForceTLBFlush = false;
1622	return;
1623	}
1624
1625
1626	/**
1627	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1628	*
1629	* @param pVM Pointer to the VM.
1630	* @param pVCpu Pointer to the VMCPU.
1631	* @param pCpu Pointer to the global HM CPU struct.
1632	* @remarks All references to "ASID" in this function pertains to "VPID" in
1633	* Intel's nomenclature. The reason is, to avoid confusion in compare
1634	* statements since the host-CPU copies are named "ASID".
1635	*
1636	* @remarks Called with interrupts disabled.
1637	*/
1638	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1639	{
1640	#ifdef VBOX_WITH_STATISTICS
1641	bool fTlbFlushed = false;
1642	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1643	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1644	if (!fTlbFlushed) \
1645	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1646	} while (0)
1647	#else
1648	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1649	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1650	#endif
1651
1652	AssertPtr(pVM);
1653	AssertPtr(pCpu);
1654	AssertPtr(pVCpu);
1655	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1656	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1657	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1658
1659
1660	/*
1661	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1662	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1663	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1664	*/
1665	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1666	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1667	{
1668	++pCpu->uCurrentAsid;
1669	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1670	{
1671	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1672	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1673	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1674	}
1675
1676	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1677	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1678	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1679
1680	/*
1681	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1682	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1683	*/
1684	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1685	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1686	HMVMX_SET_TAGGED_TLB_FLUSHED();
1687	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1688	}
1689
1690	/* Check for explicit TLB shootdowns. */
1691	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1692	{
1693	/*
1694	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1695	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1696	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1697	* but not guest-physical mappings.
1698	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1699	*/
1700	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1701	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1702	HMVMX_SET_TAGGED_TLB_FLUSHED();
1703	}
1704
1705	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1706	* where it is commented out. Support individual entry flushing
1707	* someday. */
1708	#if 0
1709	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1710	{
1711	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1712
1713	/*
1714	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1715	* as supported by the CPU.
1716	*/
1717	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1718	{
1719	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1720	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1721	}
1722	else
1723	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1724
1725	HMVMX_SET_TAGGED_TLB_FLUSHED();
1726	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1727	pVCpu->hm.s.TlbShootdown.cPages = 0;
1728	}
1729	#endif
1730
1731	pVCpu->hm.s.fForceTLBFlush = false;
1732
1733	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1734
1735	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1736	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1737	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1738	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1739	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1740	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1741	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1742	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1743
1744	/* Update VMCS with the VPID. */
1745	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1746	AssertRC(rc);
1747
1748	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1749	}
1750
1751
1752	/**
1753	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1754	*
1755	* @returns VBox status code.
1756	* @param pVM Pointer to the VM.
1757	* @param pVCpu Pointer to the VMCPU.
1758	* @param pCpu Pointer to the global HM CPU struct.
1759	*
1760	* @remarks Called with interrupts disabled.
1761	*/
1762	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1763	{
1764	AssertPtr(pVM);
1765	AssertPtr(pVCpu);
1766	AssertPtr(pCpu);
1767	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1768	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1769
1770	/*
1771	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1772	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1773	*/
1774	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1775	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1776	{
1777	pVCpu->hm.s.fForceTLBFlush = true;
1778	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1779	}
1780
1781	/* Check for explicit TLB shootdown flushes. */
1782	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1783	{
1784	pVCpu->hm.s.fForceTLBFlush = true;
1785	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1786	}
1787
1788	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1789	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1790
1791	if (pVCpu->hm.s.fForceTLBFlush)
1792	{
1793	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1794	pVCpu->hm.s.fForceTLBFlush = false;
1795	}
1796	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1797	* where it is commented out. Support individual entry flushing
1798	* someday. */
1799	#if 0
1800	else
1801	{
1802	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1803	{
1804	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1805	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1806	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1807	}
1808	else
1809	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1810
1811	pVCpu->hm.s.TlbShootdown.cPages = 0;
1812	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1813	}
1814	#endif
1815	}
1816
1817
1818	/**
1819	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1820	*
1821	* @returns VBox status code.
1822	* @param pVM Pointer to the VM.
1823	* @param pVCpu Pointer to the VMCPU.
1824	* @param pCpu Pointer to the global HM CPU struct.
1825	*
1826	* @remarks Called with interrupts disabled.
1827	*/
1828	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1829	{
1830	AssertPtr(pVM);
1831	AssertPtr(pVCpu);
1832	AssertPtr(pCpu);
1833	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1834	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1835
1836	/*
1837	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1838	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1839	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1840	*/
1841	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1842	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1843	{
1844	pVCpu->hm.s.fForceTLBFlush = true;
1845	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1846	}
1847
1848	/* Check for explicit TLB shootdown flushes. */
1849	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1850	{
1851	/*
1852	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1853	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1854	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1855	*/
1856	pVCpu->hm.s.fForceTLBFlush = true;
1857	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1858	}
1859
1860	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1861	if (pVCpu->hm.s.fForceTLBFlush)
1862	{
1863	++pCpu->uCurrentAsid;
1864	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1865	{
1866	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1867	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1868	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1869	}
1870
1871	pVCpu->hm.s.fForceTLBFlush = false;
1872	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1873	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1874	if (pCpu->fFlushAsidBeforeUse)
1875	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1876	}
1877	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1878	* where it is commented out. Support individual entry flushing
1879	* someday. */
1880	#if 0
1881	else
1882	{
1883	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1884	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1885	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1886	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1887
1888	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1889	{
1890	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1891	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1892	{
1893	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1894	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1895	}
1896	else
1897	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1898
1899	pVCpu->hm.s.TlbShootdown.cPages = 0;
1900	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1901	}
1902	else
1903	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1904	}
1905	#endif
1906
1907	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1908	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1909	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1910	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1911	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1912	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1913
1914	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1915	AssertRC(rc);
1916	}
1917
1918
1919	/**
1920	* Flushes the guest TLB entry based on CPU capabilities.
1921	*
1922	* @param pVCpu Pointer to the VMCPU.
1923	* @param pCpu Pointer to the global HM CPU struct.
1924	*/
1925	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1926	{
1927	#ifdef HMVMX_ALWAYS_FLUSH_TLB
1928	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1929	#endif
1930	PVM pVM = pVCpu->CTX_SUFF(pVM);
1931	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1932	{
1933	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1934	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1935	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1936	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1937	default:
1938	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1939	break;
1940	}
1941
1942	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
1943	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
1944
1945	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
1946	}
1947
1948
1949	/**
1950	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1951	* TLB entries from the host TLB before VM-entry.
1952	*
1953	* @returns VBox status code.
1954	* @param pVM Pointer to the VM.
1955	*/
1956	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1957	{
1958	/*
1959	* Determine optimal flush type for Nested Paging.
1960	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1961	* guest execution (see hmR3InitFinalizeR0()).
1962	*/
1963	if (pVM->hm.s.fNestedPaging)
1964	{
1965	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1966	{
1967	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1968	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1969	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1970	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1971	else
1972	{
1973	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1974	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1975	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1976	}
1977
1978	/* Make sure the write-back cacheable memory type for EPT is supported. */
1979	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1980	{
1981	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1982	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1983	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1984	}
1985	}
1986	else
1987	{
1988	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1989	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1990	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1991	}
1992	}
1993
1994	/*
1995	* Determine optimal flush type for VPID.
1996	*/
1997	if (pVM->hm.s.vmx.fVpid)
1998	{
1999	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2000	{
2001	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2002	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2003	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2004	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2005	else
2006	{
2007	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2008	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2009	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2010	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2011	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2012	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2013	pVM->hm.s.vmx.fVpid = false;
2014	}
2015	}
2016	else
2017	{
2018	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2019	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2020	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2021	pVM->hm.s.vmx.fVpid = false;
2022	}
2023	}
2024
2025	/*
2026	* Setup the handler for flushing tagged-TLBs.
2027	*/
2028	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2029	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2030	else if (pVM->hm.s.fNestedPaging)
2031	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2032	else if (pVM->hm.s.vmx.fVpid)
2033	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2034	else
2035	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2036	return VINF_SUCCESS;
2037	}
2038
2039
2040	/**
2041	* Sets up pin-based VM-execution controls in the VMCS.
2042	*
2043	* @returns VBox status code.
2044	* @param pVM Pointer to the VM.
2045	* @param pVCpu Pointer to the VMCPU.
2046	*/
2047	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2048	{
2049	AssertPtr(pVM);
2050	AssertPtr(pVCpu);
2051
2052	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2053	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2054
2055	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2056	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2057	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2058
2059	/* Enable the VMX preemption timer. */
2060	if (pVM->hm.s.vmx.fUsePreemptTimer)
2061	{
2062	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2063	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2064	}
2065
2066	if ((val & zap) != val)
2067	{
2068	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2069	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2070	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2071	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2072	}
2073
2074	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2075	AssertRCReturn(rc, rc);
2076
2077	/* Update VCPU with the currently set pin-based VM-execution controls. */
2078	pVCpu->hm.s.vmx.u32PinCtls = val;
2079	return rc;
2080	}
2081
2082
2083	/**
2084	* Sets up processor-based VM-execution controls in the VMCS.
2085	*
2086	* @returns VBox status code.
2087	* @param pVM Pointer to the VM.
2088	* @param pVMCPU Pointer to the VMCPU.
2089	*/
2090	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2091	{
2092	AssertPtr(pVM);
2093	AssertPtr(pVCpu);
2094
2095	int rc = VERR_INTERNAL_ERROR_5;
2096	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2097	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2098
2099	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2100	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2101	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2102	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2103	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2104	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2105	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2106
2107	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2108	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2109	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2110	{
2111	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2112	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2113	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2114	}
2115
2116	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2117	if (!pVM->hm.s.fNestedPaging)
2118	{
2119	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2120	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2121	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2122	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2123	}
2124
2125	/* Use TPR shadowing if supported by the CPU. */
2126	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2127	{
2128	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2129	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2130	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2131	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2132	AssertRCReturn(rc, rc);
2133
2134	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2135	/* CR8 writes causes a VM-exit based on TPR threshold. */
2136	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2137	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2138	}
2139	else
2140	{
2141	/*
2142	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2143	* Set this control only for 64-bit guests.
2144	*/
2145	if (pVM->hm.s.fAllow64BitGuests)
2146	{
2147	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2148	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2149	}
2150	}
2151
2152	/* Use MSR-bitmaps if supported by the CPU. */
2153	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2154	{
2155	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2156
2157	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2158	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2159	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2160	AssertRCReturn(rc, rc);
2161
2162	/*
2163	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2164	* automatically as dedicated fields in the VMCS.
2165	*/
2166	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2167	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2168	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2169	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2170	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2171	}
2172
2173	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2174	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2175	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2176
2177	if ((val & zap) != val)
2178	{
2179	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2180	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2181	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2182	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2183	}
2184
2185	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2186	AssertRCReturn(rc, rc);
2187
2188	/* Update VCPU with the currently set processor-based VM-execution controls. */
2189	pVCpu->hm.s.vmx.u32ProcCtls = val;
2190
2191	/*
2192	* Secondary processor-based VM-execution controls.
2193	*/
2194	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2195	{
2196	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2197	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2198
2199	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2200	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2201
2202	if (pVM->hm.s.fNestedPaging)
2203	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2204	else
2205	{
2206	/*
2207	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2208	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2209	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2210	*/
2211	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2212	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2213	}
2214
2215	if (pVM->hm.s.vmx.fVpid)
2216	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2217
2218	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2219	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2220
2221	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2222	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2223	* done dynamically. */
2224	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2225	{
2226	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2227	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2228	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2229	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2230	AssertRCReturn(rc, rc);
2231	}
2232
2233	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2234	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2235
2236	if ((val & zap) != val)
2237	{
2238	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2239	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2240	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2241	}
2242
2243	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2244	AssertRCReturn(rc, rc);
2245
2246	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2247	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2248	}
2249	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2250	{
2251	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2252	"available\n"));
2253	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254	}
2255
2256	return VINF_SUCCESS;
2257	}
2258
2259
2260	/**
2261	* Sets up miscellaneous (everything other than Pin & Processor-based
2262	* VM-execution) control fields in the VMCS.
2263	*
2264	* @returns VBox status code.
2265	* @param pVM Pointer to the VM.
2266	* @param pVCpu Pointer to the VMCPU.
2267	*/
2268	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2269	{
2270	NOREF(pVM);
2271	AssertPtr(pVM);
2272	AssertPtr(pVCpu);
2273
2274	int rc = VERR_GENERAL_FAILURE;
2275
2276	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2277	#if 0
2278	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2279	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2280	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2281
2282	/*
2283	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2284	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2285	* We thus use the exception bitmap to control it rather than use both.
2286	*/
2287	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2288	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2289
2290	/** @todo Explore possibility of using IO-bitmaps. */
2291	/* All IO & IOIO instructions cause VM-exits. */
2292	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2293	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2294
2295	/* Initialize the MSR-bitmap area. */
2296	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2297	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2298	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2299	#endif
2300
2301	/* Setup MSR auto-load/store area. */
2302	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2303	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2304	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2305	AssertRCReturn(rc, rc);
2306	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2307	AssertRCReturn(rc, rc);
2308
2309	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2310	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2311	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2312	AssertRCReturn(rc, rc);
2313
2314	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2315	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2316	AssertRCReturn(rc, rc);
2317
2318	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2319	#if 0
2320	/* Setup debug controls */
2321	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2322	AssertRCReturn(rc, rc);
2323	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2324	AssertRCReturn(rc, rc);
2325	#endif
2326
2327	return rc;
2328	}
2329
2330
2331	/**
2332	* Sets up the initial exception bitmap in the VMCS based on static conditions
2333	* (i.e. conditions that cannot ever change after starting the VM).
2334	*
2335	* @returns VBox status code.
2336	* @param pVM Pointer to the VM.
2337	* @param pVCpu Pointer to the VMCPU.
2338	*/
2339	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2340	{
2341	AssertPtr(pVM);
2342	AssertPtr(pVCpu);
2343
2344	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2345
2346	uint32_t u32XcptBitmap = 0;
2347
2348	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2349	if (!pVM->hm.s.fNestedPaging)
2350	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2351
2352	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2353	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2354	AssertRCReturn(rc, rc);
2355	return rc;
2356	}
2357
2358
2359	/**
2360	* Sets up the initial guest-state mask. The guest-state mask is consulted
2361	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2362	* for the nested virtualization case (as it would cause a VM-exit).
2363	*
2364	* @param pVCpu Pointer to the VMCPU.
2365	*/
2366	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2367	{
2368	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2369	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2370	return VINF_SUCCESS;
2371	}
2372
2373
2374	/**
2375	* Does per-VM VT-x initialization.
2376	*
2377	* @returns VBox status code.
2378	* @param pVM Pointer to the VM.
2379	*/
2380	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2381	{
2382	LogFlowFunc(("pVM=%p\n", pVM));
2383
2384	int rc = hmR0VmxStructsAlloc(pVM);
2385	if (RT_FAILURE(rc))
2386	{
2387	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2388	return rc;
2389	}
2390
2391	return VINF_SUCCESS;
2392	}
2393
2394
2395	/**
2396	* Does per-VM VT-x termination.
2397	*
2398	* @returns VBox status code.
2399	* @param pVM Pointer to the VM.
2400	*/
2401	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2402	{
2403	LogFlowFunc(("pVM=%p\n", pVM));
2404
2405	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2406	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2407	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2408	#endif
2409	hmR0VmxStructsFree(pVM);
2410	return VINF_SUCCESS;
2411	}
2412
2413
2414	/**
2415	* Sets up the VM for execution under VT-x.
2416	* This function is only called once per-VM during initialization.
2417	*
2418	* @returns VBox status code.
2419	* @param pVM Pointer to the VM.
2420	*/
2421	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2422	{
2423	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2424	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2425
2426	LogFlowFunc(("pVM=%p\n", pVM));
2427
2428	/*
2429	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2430	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2431	*/
2432	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2433	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2434	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2435	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2436	{
2437	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2438	return VERR_INTERNAL_ERROR;
2439	}
2440
2441	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2442	/*
2443	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2444	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2445	*/
2446	if ( pVM->hm.s.fAllow64BitGuests
2447	&& !HMVMX_IS_64BIT_HOST_MODE())
2448	{
2449	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2450	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2451	}
2452	#endif
2453
2454	/* Initialize these always, see hmR3InitFinalizeR0().*/
2455	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2456	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2457
2458	/* Setup the tagged-TLB flush handlers. */
2459	int rc = hmR0VmxSetupTaggedTlb(pVM);
2460	if (RT_FAILURE(rc))
2461	{
2462	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2463	return rc;
2464	}
2465
2466	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2467	{
2468	PVMCPU pVCpu = &pVM->aCpus[i];
2469	AssertPtr(pVCpu);
2470	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2471
2472	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2473	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2474
2475	/* Set revision dword at the beginning of the VMCS structure. */
2476	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2477
2478	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2479	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2480	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2481	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2482
2483	/* Load this VMCS as the current VMCS. */
2484	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2485	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2486	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2487
2488	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2489	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2490	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2491
2492	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2493	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2494	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2495
2496	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2497	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2498	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2499
2500	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2501	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2502	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2503
2504	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2505	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2506	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2507
2508	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2509	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2510	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2511	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2512	#endif
2513
2514	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2515	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2516	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2517	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2518
2519	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2520
2521	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2522	}
2523
2524	return VINF_SUCCESS;
2525	}
2526
2527
2528	/**
2529	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2530	* the VMCS.
2531	*
2532	* @returns VBox status code.
2533	* @param pVM Pointer to the VM.
2534	* @param pVCpu Pointer to the VMCPU.
2535	*/
2536	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2537	{
2538	NOREF(pVM); NOREF(pVCpu);
2539
2540	RTCCUINTREG uReg = ASMGetCR0();
2541	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2542	AssertRCReturn(rc, rc);
2543
2544	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2545	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2546	if (HMVMX_IS_64BIT_HOST_MODE())
2547	{
2548	uint64_t uRegCR3 = HMR0Get64bitCR3();
2549	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2550	}
2551	else
2552	#endif
2553	{
2554	uReg = ASMGetCR3();
2555	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2556	}
2557	AssertRCReturn(rc, rc);
2558
2559	uReg = ASMGetCR4();
2560	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2561	AssertRCReturn(rc, rc);
2562	return rc;
2563	}
2564
2565
2566	#if HC_ARCH_BITS == 64
2567	/**
2568	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2569	* requirements. See hmR0VmxSaveHostSegmentRegs().
2570	*/
2571	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2572	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2573	{ \
2574	bool fValidSelector = true; \
2575	if ((selValue) & X86_SEL_LDT) \
2576	{ \
2577	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2578	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2579	} \
2580	if (fValidSelector) \
2581	{ \
2582	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2583	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2584	} \
2585	(selValue) = 0; \
2586	}
2587	#endif
2588
2589
2590	/**
2591	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2592	* the host-state area in the VMCS.
2593	*
2594	* @returns VBox status code.
2595	* @param pVM Pointer to the VM.
2596	* @param pVCpu Pointer to the VMCPU.
2597	*/
2598	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2599	{
2600	NOREF(pVM);
2601	int rc = VERR_INTERNAL_ERROR_5;
2602
2603	/*
2604	* Host DS, ES, FS and GS segment registers.
2605	*/
2606	#if HC_ARCH_BITS == 64
2607	RTSEL uSelDS = ASMGetDS();
2608	RTSEL uSelES = ASMGetES();
2609	RTSEL uSelFS = ASMGetFS();
2610	RTSEL uSelGS = ASMGetGS();
2611	#else
2612	RTSEL uSelDS = 0;
2613	RTSEL uSelES = 0;
2614	RTSEL uSelFS = 0;
2615	RTSEL uSelGS = 0;
2616	#endif
2617
2618	/* Recalculate which host-state bits need to be manually restored. */
2619	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2620
2621	/*
2622	* Host CS and SS segment registers.
2623	*/
2624	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2625	RTSEL uSelCS;
2626	RTSEL uSelSS;
2627	if (HMVMX_IS_64BIT_HOST_MODE())
2628	{
2629	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2630	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2631	}
2632	else
2633	{
2634	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2635	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2636	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2637	}
2638	#else
2639	RTSEL uSelCS = ASMGetCS();
2640	RTSEL uSelSS = ASMGetSS();
2641	#endif
2642
2643	/*
2644	* Host TR segment register.
2645	*/
2646	RTSEL uSelTR = ASMGetTR();
2647
2648	#if HC_ARCH_BITS == 64
2649	/*
2650	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2651	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2652	*/
2653	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2654	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2655	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2656	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2657	# undef VMXLOCAL_ADJUST_HOST_SEG
2658	#endif
2659
2660	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2661	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2662	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2663	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2664	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2665	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2666	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2667	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2668	Assert(uSelCS);
2669	Assert(uSelTR);
2670
2671	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2672	#if 0
2673	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2674	Assert(uSelSS != 0);
2675	#endif
2676
2677	/* Write these host selector fields into the host-state area in the VMCS. */
2678	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2679	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2680	#if HC_ARCH_BITS == 64
2681	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2682	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2683	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2684	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2685	#endif
2686	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2687
2688	/*
2689	* Host GDTR and IDTR.
2690	*/
2691	RTGDTR Gdtr;
2692	RT_ZERO(Gdtr);
2693	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2694	if (HMVMX_IS_64BIT_HOST_MODE())
2695	{
2696	X86XDTR64 Gdtr64;
2697	X86XDTR64 Idtr64;
2698	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2699	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2700	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2701
2702	Gdtr.cbGdt = Gdtr64.cb;
2703	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2704	}
2705	else
2706	#endif
2707	{
2708	RTIDTR Idtr;
2709	ASMGetGDTR(&Gdtr);
2710	ASMGetIDTR(&Idtr);
2711	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2712	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2713
2714	#if HC_ARCH_BITS == 64
2715	/*
2716	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2717	* maximum limit (0xffff) on every VM-exit.
2718	*/
2719	if (Gdtr.cbGdt != 0xffff)
2720	{
2721	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2722	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2723	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2724	}
2725
2726	/*
2727	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2728	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2729	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2730	*/
2731	if (Idtr.cbIdt < 0x0fff)
2732	{
2733	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2734	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2735	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2736	}
2737	#endif
2738	}
2739
2740	/*
2741	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2742	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2743	*/
2744	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2745	{
2746	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2747	return VERR_VMX_INVALID_HOST_STATE;
2748	}
2749
2750	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2751	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2752	if (HMVMX_IS_64BIT_HOST_MODE())
2753	{
2754	/* We need the 64-bit TR base for hybrid darwin. */
2755	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2756	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2757	}
2758	else
2759	#endif
2760	{
2761	uintptr_t uTRBase;
2762	#if HC_ARCH_BITS == 64
2763	uTRBase = X86DESC64_BASE(pDesc);
2764
2765	/*
2766	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2767	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2768	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2769	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2770	*
2771	* [1] See Intel spec. 3.5 "System Descriptor Types".
2772	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2773	*/
2774	Assert(pDesc->System.u4Type == 11);
2775	if ( pDesc->System.u16LimitLow != 0x67
2776	\|\| pDesc->System.u4LimitHigh)
2777	{
2778	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2779	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2780
2781	/* Store the GDTR here as we need it while restoring TR. */
2782	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2783	}
2784	#else
2785	uTRBase = X86DESC_BASE(pDesc);
2786	#endif
2787	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2788	}
2789	AssertRCReturn(rc, rc);
2790
2791	/*
2792	* Host FS base and GS base.
2793	*/
2794	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2795	if (HMVMX_IS_64BIT_HOST_MODE())
2796	{
2797	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2798	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2799	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2800	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2801
2802	# if HC_ARCH_BITS == 64
2803	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2804	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2805	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2806	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2807	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2808	# endif
2809	}
2810	#endif
2811	return rc;
2812	}
2813
2814
2815	/**
2816	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2817	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2818	* the host after every successful VM exit.
2819	*
2820	* @returns VBox status code.
2821	* @param pVM Pointer to the VM.
2822	* @param pVCpu Pointer to the VMCPU.
2823	*/
2824	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2825	{
2826	NOREF(pVM);
2827
2828	AssertPtr(pVCpu);
2829	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2830
2831	int rc = VINF_SUCCESS;
2832	#if 0
2833	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2834	uint32_t cHostMsrs = 0;
2835	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2836
2837	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2838	{
2839	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2840
2841	# if HC_ARCH_BITS == 64
2842	/* Paranoia. 64-bit code requires these bits to be set always. */
2843	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2844
2845	/*
2846	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2847	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2848	* some reason (e.g. allow transparent reads) we would activate the code below.
2849	*/
2850	# if 0
2851	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2852	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2853	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2854	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2855	if (CPUMIsGuestInLongMode(pVCpu))
2856	{
2857	uint64_t u64GuestEfer;
2858	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2859	AssertRC(rc);
2860
2861	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2862	{
2863	pHostMsr->u32Msr = MSR_K6_EFER;
2864	pHostMsr->u32Reserved = 0;
2865	pHostMsr->u64Value = u64HostEfer;
2866	pHostMsr++; cHostMsrs++;
2867	}
2868	}
2869	# endif
2870	# else /* HC_ARCH_BITS != 64 */
2871	pHostMsr->u32Msr = MSR_K6_EFER;
2872	pHostMsr->u32Reserved = 0;
2873	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2874	if (CPUMIsGuestInLongMode(pVCpu))
2875	{
2876	/* Must match the EFER value in our 64-bit switcher. */
2877	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2878	}
2879	else
2880	# endif
2881	pHostMsr->u64Value = u64HostEfer;
2882	pHostMsr++; cHostMsrs++;
2883	# endif /* HC_ARCH_BITS == 64 */
2884	}
2885
2886	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2887	if (HMVMX_IS_64BIT_HOST_MODE())
2888	{
2889	pHostMsr->u32Msr = MSR_K6_STAR;
2890	pHostMsr->u32Reserved = 0;
2891	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2892	pHostMsr++; cHostMsrs++;
2893	pHostMsr->u32Msr = MSR_K8_LSTAR;
2894	pHostMsr->u32Reserved = 0;
2895	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2896	pHostMsr++; cHostMsrs++;
2897	pHostMsr->u32Msr = MSR_K8_SF_MASK;
2898	pHostMsr->u32Reserved = 0;
2899	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2900	pHostMsr++; cHostMsrs++;
2901	pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2902	pHostMsr->u32Reserved = 0;
2903	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2904	pHostMsr++; cHostMsrs++;
2905	}
2906	# endif
2907
2908	/* Host TSC AUX MSR must be restored since we always load/store guest TSC AUX MSR. */
2909	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2910	{
2911	pHostMsr->u32Msr = MSR_K8_TSC_AUX;
2912	pHostMsr->u32Reserved = 0;
2913	pHostMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
2914	pHostMsr++; cHostMsrs++;
2915	}
2916
2917	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2918	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2919	{
2920	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2921	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2922	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2923	}
2924
2925	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2926	#endif
2927
2928	if (pVCpu->hm.s.vmx.cMsrs > 0)
2929	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
2930
2931	/*
2932	* Host Sysenter MSRs.
2933	*/
2934	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2935	AssertRCReturn(rc, rc);
2936	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2937	if (HMVMX_IS_64BIT_HOST_MODE())
2938	{
2939	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2940	AssertRCReturn(rc, rc);
2941	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2942	}
2943	else
2944	{
2945	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2946	AssertRCReturn(rc, rc);
2947	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2948	}
2949	#elif HC_ARCH_BITS == 32
2950	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2951	AssertRCReturn(rc, rc);
2952	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2953	#else
2954	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2955	AssertRCReturn(rc, rc);
2956	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2957	#endif
2958	AssertRCReturn(rc, rc);
2959
2960	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2961	* hmR0VmxSetupExitCtls() !! */
2962	return rc;
2963	}
2964
2965
2966	/**
2967	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2968	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2969	* controls".
2970	*
2971	* @returns VBox status code.
2972	* @param pVCpu Pointer to the VMCPU.
2973	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2974	* out-of-sync. Make sure to update the required fields
2975	* before using them.
2976	*
2977	* @remarks No-long-jump zone!!!
2978	*/
2979	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2980	{
2981	int rc = VINF_SUCCESS;
2982	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2983	{
2984	PVM pVM = pVCpu->CTX_SUFF(pVM);
2985	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2986	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2987
2988	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2989	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2990
2991	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2992	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2993	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2994	else
2995	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2996
2997	/*
2998	* The following should -not- be set (since we're not in SMM mode):
2999	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3000	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3001	*/
3002
3003	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3004	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3005	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3006
3007	if ((val & zap) != val)
3008	{
3009	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3010	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3011	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3012	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3013	}
3014
3015	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3016	AssertRCReturn(rc, rc);
3017
3018	/* Update VCPU with the currently set VM-exit controls. */
3019	pVCpu->hm.s.vmx.u32EntryCtls = val;
3020	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3021	}
3022	return rc;
3023	}
3024
3025
3026	/**
3027	* Sets up the VM-exit controls in the VMCS.
3028	*
3029	* @returns VBox status code.
3030	* @param pVM Pointer to the VM.
3031	* @param pVCpu Pointer to the VMCPU.
3032	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3033	* out-of-sync. Make sure to update the required fields
3034	* before using them.
3035	*
3036	* @remarks requires EFER.
3037	*/
3038	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3039	{
3040	NOREF(pMixedCtx);
3041
3042	int rc = VINF_SUCCESS;
3043	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3044	{
3045	PVM pVM = pVCpu->CTX_SUFF(pVM);
3046	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3047	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3048
3049	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3050	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3051
3052	/*
3053	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3054	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3055	*/
3056	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3057	if (HMVMX_IS_64BIT_HOST_MODE())
3058	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3059	else
3060	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3061	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3062	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3063	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3064	else
3065	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3066	#endif
3067
3068	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3069	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3070
3071	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3072	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3073	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3074	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3075	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3076
3077	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3078	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3079
3080	if ((val & zap) != val)
3081	{
3082	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3083	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3084	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3085	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3086	}
3087
3088	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3089	AssertRCReturn(rc, rc);
3090
3091	/* Update VCPU with the currently set VM-exit controls. */
3092	pVCpu->hm.s.vmx.u32ExitCtls = val;
3093	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3094	}
3095	return rc;
3096	}
3097
3098
3099	/**
3100	* Loads the guest APIC and related state.
3101	*
3102	* @returns VBox status code.
3103	* @param pVM Pointer to the VM.
3104	* @param pVCpu Pointer to the VMCPU.
3105	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3106	* out-of-sync. Make sure to update the required fields
3107	* before using them.
3108	*/
3109	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3110	{
3111	NOREF(pMixedCtx);
3112
3113	int rc = VINF_SUCCESS;
3114	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3115	{
3116	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3117	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3118	{
3119	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3120
3121	bool fPendingIntr = false;
3122	uint8_t u8Tpr = 0;
3123	uint8_t u8PendingIntr = 0;
3124	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3125	AssertRCReturn(rc, rc);
3126
3127	/*
3128	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3129	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3130	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3131	* the interrupt when we VM-exit for other reasons.
3132	*/
3133	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3134	uint32_t u32TprThreshold = 0;
3135	if (fPendingIntr)
3136	{
3137	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3138	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3139	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3140	if (u8PendingPriority <= u8TprPriority)
3141	u32TprThreshold = u8PendingPriority;
3142	else
3143	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3144	}
3145	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3146
3147	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3148	AssertRCReturn(rc, rc);
3149	}
3150
3151	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3152	}
3153	return rc;
3154	}
3155
3156
3157	/**
3158	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3159	*
3160	* @returns Guest's interruptibility-state.
3161	* @param pVCpu Pointer to the VMCPU.
3162	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3163	* out-of-sync. Make sure to update the required fields
3164	* before using them.
3165	*
3166	* @remarks No-long-jump zone!!!
3167	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3168	*/
3169	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3170	{
3171	/*
3172	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3173	* inhibit interrupts or clear any existing interrupt-inhibition.
3174	*/
3175	uint32_t uIntrState = 0;
3176	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3177	{
3178	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3179	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
3180	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3181	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3182	{
3183	/*
3184	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3185	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3186	*/
3187	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3188	}
3189	else if (pMixedCtx->eflags.Bits.u1IF)
3190	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3191	else
3192	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3193	}
3194	return uIntrState;
3195	}
3196
3197
3198	/**
3199	* Loads the guest's interruptibility-state into the guest-state area in the
3200	* VMCS.
3201	*
3202	* @returns VBox status code.
3203	* @param pVCpu Pointer to the VMCPU.
3204	* @param uIntrState The interruptibility-state to set.
3205	*/
3206	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3207	{
3208	NOREF(pVCpu);
3209	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3210	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3211	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3212	AssertRCReturn(rc, rc);
3213	return rc;
3214	}
3215
3216
3217	/**
3218	* Loads the guest's RIP into the guest-state area in the VMCS.
3219	*
3220	* @returns VBox status code.
3221	* @param pVCpu Pointer to the VMCPU.
3222	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3223	* out-of-sync. Make sure to update the required fields
3224	* before using them.
3225	*
3226	* @remarks No-long-jump zone!!!
3227	*/
3228	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3229	{
3230	int rc = VINF_SUCCESS;
3231	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3232	{
3233	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3234	AssertRCReturn(rc, rc);
3235
3236	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3237	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3238	}
3239	return rc;
3240	}
3241
3242
3243	/**
3244	* Loads the guest's RSP into the guest-state area in the VMCS.
3245	*
3246	* @returns VBox status code.
3247	* @param pVCpu Pointer to the VMCPU.
3248	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3249	* out-of-sync. Make sure to update the required fields
3250	* before using them.
3251	*
3252	* @remarks No-long-jump zone!!!
3253	*/
3254	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3255	{
3256	int rc = VINF_SUCCESS;
3257	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3258	{
3259	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3260	AssertRCReturn(rc, rc);
3261
3262	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3263	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3264	}
3265	return rc;
3266	}
3267
3268
3269	/**
3270	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3271	*
3272	* @returns VBox status code.
3273	* @param pVCpu Pointer to the VMCPU.
3274	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3275	* out-of-sync. Make sure to update the required fields
3276	* before using them.
3277	*
3278	* @remarks No-long-jump zone!!!
3279	*/
3280	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3281	{
3282	int rc = VINF_SUCCESS;
3283	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3284	{
3285	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3286	Let us assert it as such and use 32-bit VMWRITE. */
3287	Assert(!(pMixedCtx->rflags.u64 >> 32));
3288	X86EFLAGS Eflags = pMixedCtx->eflags;
3289	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3290	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3291
3292	/*
3293	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3294	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3295	*/
3296	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3297	{
3298	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3299	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3300	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3301	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3302	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3303	}
3304
3305	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3306	AssertRCReturn(rc, rc);
3307
3308	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3309	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3310	}
3311	return rc;
3312	}
3313
3314
3315	/**
3316	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3317	*
3318	* @returns VBox status code.
3319	* @param pVCpu Pointer to the VMCPU.
3320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3321	* out-of-sync. Make sure to update the required fields
3322	* before using them.
3323	*
3324	* @remarks No-long-jump zone!!!
3325	*/
3326	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3327	{
3328	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3329	AssertRCReturn(rc, rc);
3330	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3331	AssertRCReturn(rc, rc);
3332	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3333	AssertRCReturn(rc, rc);
3334	return rc;
3335	}
3336
3337
3338	/**
3339	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3340	* CR0 is partially shared with the host and we have to consider the FPU bits.
3341	*
3342	* @returns VBox status code.
3343	* @param pVM Pointer to the VM.
3344	* @param pVCpu Pointer to the VMCPU.
3345	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3346	* out-of-sync. Make sure to update the required fields
3347	* before using them.
3348	*
3349	* @remarks No-long-jump zone!!!
3350	*/
3351	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3352	{
3353	/*
3354	* Guest CR0.
3355	* Guest FPU.
3356	*/
3357	int rc = VINF_SUCCESS;
3358	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3359	{
3360	Assert(!(pMixedCtx->cr0 >> 32));
3361	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3362	PVM pVM = pVCpu->CTX_SUFF(pVM);
3363
3364	/* The guest's view (read access) of its CR0 is unblemished. */
3365	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3366	AssertRCReturn(rc, rc);
3367	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3368
3369	/* Setup VT-x's view of the guest CR0. */
3370	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3371	if (pVM->hm.s.fNestedPaging)
3372	{
3373	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3374	{
3375	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3376	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3377	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3378	}
3379	else
3380	{
3381	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3382	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3383	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3384	}
3385
3386	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3387	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3388	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3389
3390	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3391	AssertRCReturn(rc, rc);
3392	}
3393	else
3394	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3395
3396	/*
3397	* Guest FPU bits.
3398	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3399	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3400	*/
3401	u32GuestCR0 \|= X86_CR0_NE;
3402	bool fInterceptNM = false;
3403	if (CPUMIsGuestFPUStateActive(pVCpu))
3404	{
3405	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3406	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3407	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3408	}
3409	else
3410	{
3411	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3412	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3413	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3414	}
3415
3416	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3417	bool fInterceptMF = false;
3418	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3419	fInterceptMF = true;
3420
3421	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3422	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3423	{
3424	Assert(PDMVmmDevHeapIsEnabled(pVM));
3425	Assert(pVM->hm.s.vmx.pRealModeTSS);
3426	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3427	fInterceptNM = true;
3428	fInterceptMF = true;
3429	}
3430	else
3431	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3432
3433	if (fInterceptNM)
3434	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3435	else
3436	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3437
3438	if (fInterceptMF)
3439	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3440	else
3441	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3442
3443	/* Additional intercepts for debugging, define these yourself explicitly. */
3444	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3445	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3446	\| RT_BIT(X86_XCPT_BP)
3447	\| RT_BIT(X86_XCPT_DB)
3448	\| RT_BIT(X86_XCPT_DE)
3449	\| RT_BIT(X86_XCPT_NM)
3450	\| RT_BIT(X86_XCPT_UD)
3451	\| RT_BIT(X86_XCPT_NP)
3452	\| RT_BIT(X86_XCPT_SS)
3453	\| RT_BIT(X86_XCPT_GP)
3454	\| RT_BIT(X86_XCPT_PF)
3455	\| RT_BIT(X86_XCPT_MF)
3456	;
3457	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3458	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3459	#endif
3460
3461	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3462
3463	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3464	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3465	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3466	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3467	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3468	else
3469	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3470
3471	u32GuestCR0 \|= uSetCR0;
3472	u32GuestCR0 &= uZapCR0;
3473	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3474
3475	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3476	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3477	AssertRCReturn(rc, rc);
3478	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3479	AssertRCReturn(rc, rc);
3480	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3481
3482	/*
3483	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3484	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3485	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3486	*/
3487	uint32_t u32CR0Mask = 0;
3488	u32CR0Mask = X86_CR0_PE
3489	\| X86_CR0_NE
3490	\| X86_CR0_WP
3491	\| X86_CR0_PG
3492	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3493	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3494	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3495
3496	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3497	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3498	* and @bugref{6944}. */
3499	#if 0
3500	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3501	u32CR0Mask &= ~X86_CR0_PE;
3502	#endif
3503	if (pVM->hm.s.fNestedPaging)
3504	u32CR0Mask &= ~X86_CR0_WP;
3505
3506	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3507	if (fInterceptNM)
3508	{
3509	u32CR0Mask \|= X86_CR0_TS
3510	\| X86_CR0_MP;
3511	}
3512
3513	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3514	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3515	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3516	AssertRCReturn(rc, rc);
3517	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3518
3519	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3520	}
3521	return rc;
3522	}
3523
3524
3525	/**
3526	* Loads the guest control registers (CR3, CR4) into the guest-state area
3527	* in the VMCS.
3528	*
3529	* @returns VBox status code.
3530	* @param pVM Pointer to the VM.
3531	* @param pVCpu Pointer to the VMCPU.
3532	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3533	* out-of-sync. Make sure to update the required fields
3534	* before using them.
3535	*
3536	* @remarks No-long-jump zone!!!
3537	*/
3538	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3539	{
3540	int rc = VINF_SUCCESS;
3541	PVM pVM = pVCpu->CTX_SUFF(pVM);
3542
3543	/*
3544	* Guest CR2.
3545	* It's always loaded in the assembler code. Nothing to do here.
3546	*/
3547
3548	/*
3549	* Guest CR3.
3550	*/
3551	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3552	{
3553	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3554	if (pVM->hm.s.fNestedPaging)
3555	{
3556	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3557
3558	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3559	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3560	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3561	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3562
3563	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3564	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3565	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3566
3567	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3568	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3569	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3570	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3571
3572	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3573	AssertRCReturn(rc, rc);
3574	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3575
3576	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3577	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3578	{
3579	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3580	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3581	{
3582	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3583	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3584	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3585	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3586	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3587	}
3588
3589	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3590	have Unrestricted Execution to handle the guest when it's not using paging. */
3591	GCPhysGuestCR3 = pMixedCtx->cr3;
3592	}
3593	else
3594	{
3595	/*
3596	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3597	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3598	* EPT takes care of translating it to host-physical addresses.
3599	*/
3600	RTGCPHYS GCPhys;
3601	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3602	Assert(PDMVmmDevHeapIsEnabled(pVM));
3603
3604	/* We obtain it here every time as the guest could have relocated this PCI region. */
3605	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3606	AssertRCReturn(rc, rc);
3607
3608	GCPhysGuestCR3 = GCPhys;
3609	}
3610
3611	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3612	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3613	}
3614	else
3615	{
3616	/* Non-nested paging case, just use the hypervisor's CR3. */
3617	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3618
3619	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3620	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3621	}
3622	AssertRCReturn(rc, rc);
3623
3624	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3625	}
3626
3627	/*
3628	* Guest CR4.
3629	*/
3630	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3631	{
3632	Assert(!(pMixedCtx->cr4 >> 32));
3633	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3634
3635	/* The guest's view of its CR4 is unblemished. */
3636	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3637	AssertRCReturn(rc, rc);
3638	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3639
3640	/* Setup VT-x's view of the guest CR4. */
3641	/*
3642	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3643	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3644	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3645	*/
3646	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3647	{
3648	Assert(pVM->hm.s.vmx.pRealModeTSS);
3649	Assert(PDMVmmDevHeapIsEnabled(pVM));
3650	u32GuestCR4 &= ~X86_CR4_VME;
3651	}
3652
3653	if (pVM->hm.s.fNestedPaging)
3654	{
3655	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3656	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3657	{
3658	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3659	u32GuestCR4 \|= X86_CR4_PSE;
3660	/* Our identity mapping is a 32-bit page directory. */
3661	u32GuestCR4 &= ~X86_CR4_PAE;
3662	}
3663	/* else use guest CR4.*/
3664	}
3665	else
3666	{
3667	/*
3668	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3669	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3670	*/
3671	switch (pVCpu->hm.s.enmShadowMode)
3672	{
3673	case PGMMODE_REAL: /* Real-mode. */
3674	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3675	case PGMMODE_32_BIT: /* 32-bit paging. */
3676	{
3677	u32GuestCR4 &= ~X86_CR4_PAE;
3678	break;
3679	}
3680
3681	case PGMMODE_PAE: /* PAE paging. */
3682	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3683	{
3684	u32GuestCR4 \|= X86_CR4_PAE;
3685	break;
3686	}
3687
3688	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3689	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3690	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3691	break;
3692	#endif
3693	default:
3694	AssertFailed();
3695	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3696	}
3697	}
3698
3699	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3700	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3701	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3702	u32GuestCR4 \|= uSetCR4;
3703	u32GuestCR4 &= uZapCR4;
3704
3705	/* Write VT-x's view of the guest CR4 into the VMCS. */
3706	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3707	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3708	AssertRCReturn(rc, rc);
3709
3710	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3711	uint32_t u32CR4Mask = 0;
3712	u32CR4Mask = X86_CR4_VME
3713	\| X86_CR4_PAE
3714	\| X86_CR4_PGE
3715	\| X86_CR4_PSE
3716	\| X86_CR4_VMXE;
3717	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3718	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3719	AssertRCReturn(rc, rc);
3720
3721	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3722	}
3723	return rc;
3724	}
3725
3726
3727	/**
3728	* Loads the guest debug registers into the guest-state area in the VMCS.
3729	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3730	*
3731	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3732	*
3733	* @returns VBox status code.
3734	* @param pVCpu Pointer to the VMCPU.
3735	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3736	* out-of-sync. Make sure to update the required fields
3737	* before using them.
3738	*
3739	* @remarks No-long-jump zone!!!
3740	*/
3741	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3742	{
3743	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3744	return VINF_SUCCESS;
3745
3746	#ifdef VBOX_STRICT
3747	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3748	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3749	{
3750	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3751	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3752	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3753	}
3754	#endif
3755
3756	int rc;
3757	PVM pVM = pVCpu->CTX_SUFF(pVM);
3758	bool fInterceptDB = false;
3759	bool fInterceptMovDRx = false;
3760	if ( pVCpu->hm.s.fSingleInstruction
3761	\|\| DBGFIsStepping(pVCpu))
3762	{
3763	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3764	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3765	{
3766	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3767	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3768	AssertRCReturn(rc, rc);
3769	Assert(fInterceptDB == false);
3770	}
3771	else
3772	{
3773	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3774	pVCpu->hm.s.fClearTrapFlag = true;
3775	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3776	fInterceptDB = true;
3777	}
3778	}
3779
3780	if ( fInterceptDB
3781	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3782	{
3783	/*
3784	* Use the combined guest and host DRx values found in the hypervisor
3785	* register set because the debugger has breakpoints active or someone
3786	* is single stepping on the host side without a monitor trap flag.
3787	*
3788	* Note! DBGF expects a clean DR6 state before executing guest code.
3789	*/
3790	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3791	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3792	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3793	{
3794	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3795	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3796	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3797	}
3798	else
3799	#endif
3800	if (!CPUMIsHyperDebugStateActive(pVCpu))
3801	{
3802	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3803	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3804	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3805	}
3806
3807	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3808	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3809	AssertRCReturn(rc, rc);
3810
3811	pVCpu->hm.s.fUsingHyperDR7 = true;
3812	fInterceptDB = true;
3813	fInterceptMovDRx = true;
3814	}
3815	else
3816	{
3817	/*
3818	* If the guest has enabled debug registers, we need to load them prior to
3819	* executing guest code so they'll trigger at the right time.
3820	*/
3821	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3822	{
3823	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3824	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3825	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3826	{
3827	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3828	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3829	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3830	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3831	}
3832	else
3833	#endif
3834	if (!CPUMIsGuestDebugStateActive(pVCpu))
3835	{
3836	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3837	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3838	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3839	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3840	}
3841	Assert(!fInterceptDB);
3842	Assert(!fInterceptMovDRx);
3843	}
3844	/*
3845	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3846	* must intercept #DB in order to maintain a correct DR6 guest value.
3847	*/
3848	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3849	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3850	&& !CPUMIsGuestDebugStateActive(pVCpu))
3851	#else
3852	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3853	#endif
3854	{
3855	fInterceptMovDRx = true;
3856	fInterceptDB = true;
3857	}
3858
3859	/* Update guest DR7. */
3860	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3861	AssertRCReturn(rc, rc);
3862
3863	pVCpu->hm.s.fUsingHyperDR7 = false;
3864	}
3865
3866	/*
3867	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3868	*/
3869	if (fInterceptDB)
3870	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3871	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3872	{
3873	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3874	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3875	#endif
3876	}
3877	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3878	AssertRCReturn(rc, rc);
3879
3880	/*
3881	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3882	*/
3883	if (fInterceptMovDRx)
3884	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3885	else
3886	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3887	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3888	AssertRCReturn(rc, rc);
3889
3890	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3891	return VINF_SUCCESS;
3892	}
3893
3894
3895	#ifdef VBOX_STRICT
3896	/**
3897	* Strict function to validate segment registers.
3898	*
3899	* @remarks ASSUMES CR0 is up to date.
3900	*/
3901	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3902	{
3903	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3904	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3905	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3906	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3907	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3908	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3909	{
3910	/* Protected mode checks */
3911	/* CS */
3912	Assert(pCtx->cs.Attr.n.u1Present);
3913	Assert(!(pCtx->cs.Attr.u & 0xf00));
3914	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3915	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3916	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3917	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3918	\|\| (pCtx->cs.Attr.n.u1Granularity));
3919	/* CS cannot be loaded with NULL in protected mode. */
3920	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3921	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3922	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3923	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3924	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3925	else
3926	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3927	/* SS */
3928	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3929	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3930	if ( !(pCtx->cr0 & X86_CR0_PE)
3931	\|\| pCtx->cs.Attr.n.u4Type == 3)
3932	{
3933	Assert(!pCtx->ss.Attr.n.u2Dpl);
3934	}
3935	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3936	{
3937	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3938	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3939	Assert(pCtx->ss.Attr.n.u1Present);
3940	Assert(!(pCtx->ss.Attr.u & 0xf00));
3941	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3942	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3943	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3944	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3945	\|\| (pCtx->ss.Attr.n.u1Granularity));
3946	}
3947	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3948	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3949	{
3950	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3951	Assert(pCtx->ds.Attr.n.u1Present);
3952	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3953	Assert(!(pCtx->ds.Attr.u & 0xf00));
3954	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3955	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3956	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3957	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3958	\|\| (pCtx->ds.Attr.n.u1Granularity));
3959	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3960	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3961	}
3962	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3963	{
3964	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3965	Assert(pCtx->es.Attr.n.u1Present);
3966	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3967	Assert(!(pCtx->es.Attr.u & 0xf00));
3968	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3969	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3970	\|\| !(pCtx->es.Attr.n.u1Granularity));
3971	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3972	\|\| (pCtx->es.Attr.n.u1Granularity));
3973	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3974	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3975	}
3976	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3977	{
3978	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3979	Assert(pCtx->fs.Attr.n.u1Present);
3980	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3981	Assert(!(pCtx->fs.Attr.u & 0xf00));
3982	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3983	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3984	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3985	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3986	\|\| (pCtx->fs.Attr.n.u1Granularity));
3987	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3988	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3989	}
3990	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3991	{
3992	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3993	Assert(pCtx->gs.Attr.n.u1Present);
3994	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3995	Assert(!(pCtx->gs.Attr.u & 0xf00));
3996	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3997	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3998	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3999	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4000	\|\| (pCtx->gs.Attr.n.u1Granularity));
4001	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4002	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4003	}
4004	/* 64-bit capable CPUs. */
4005	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4006	Assert(!(pCtx->cs.u64Base >> 32));
4007	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4008	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4009	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4010	# endif
4011	}
4012	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4013	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4014	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4015	{
4016	/* Real and v86 mode checks. */
4017	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4018	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4019	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4020	{
4021	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4022	}
4023	else
4024	{
4025	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4026	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4027	}
4028
4029	/* CS */
4030	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4031	Assert(pCtx->cs.u32Limit == 0xffff);
4032	Assert(u32CSAttr == 0xf3);
4033	/* SS */
4034	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4035	Assert(pCtx->ss.u32Limit == 0xffff);
4036	Assert(u32SSAttr == 0xf3);
4037	/* DS */
4038	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4039	Assert(pCtx->ds.u32Limit == 0xffff);
4040	Assert(u32DSAttr == 0xf3);
4041	/* ES */
4042	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4043	Assert(pCtx->es.u32Limit == 0xffff);
4044	Assert(u32ESAttr == 0xf3);
4045	/* FS */
4046	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4047	Assert(pCtx->fs.u32Limit == 0xffff);
4048	Assert(u32FSAttr == 0xf3);
4049	/* GS */
4050	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4051	Assert(pCtx->gs.u32Limit == 0xffff);
4052	Assert(u32GSAttr == 0xf3);
4053	/* 64-bit capable CPUs. */
4054	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4055	Assert(!(pCtx->cs.u64Base >> 32));
4056	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4057	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4058	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4059	# endif
4060	}
4061	}
4062	#endif /* VBOX_STRICT */
4063
4064
4065	/**
4066	* Writes a guest segment register into the guest-state area in the VMCS.
4067	*
4068	* @returns VBox status code.
4069	* @param pVCpu Pointer to the VMCPU.
4070	* @param idxSel Index of the selector in the VMCS.
4071	* @param idxLimit Index of the segment limit in the VMCS.
4072	* @param idxBase Index of the segment base in the VMCS.
4073	* @param idxAccess Index of the access rights of the segment in the VMCS.
4074	* @param pSelReg Pointer to the segment selector.
4075	*
4076	* @remarks No-long-jump zone!!!
4077	*/
4078	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4079	uint32_t idxAccess, PCPUMSELREG pSelReg)
4080	{
4081	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4082	AssertRCReturn(rc, rc);
4083	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4084	AssertRCReturn(rc, rc);
4085	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4086	AssertRCReturn(rc, rc);
4087
4088	uint32_t u32Access = pSelReg->Attr.u;
4089	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4090	{
4091	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4092	u32Access = 0xf3;
4093	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4094	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4095	}
4096	else
4097	{
4098	/*
4099	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4100	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4101	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4102	* loaded in protected-mode have their attribute as 0.
4103	*/
4104	if (!u32Access)
4105	u32Access = X86DESCATTR_UNUSABLE;
4106	}
4107
4108	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4109	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4110	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4111
4112	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4113	AssertRCReturn(rc, rc);
4114	return rc;
4115	}
4116
4117
4118	/**
4119	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4120	* into the guest-state area in the VMCS.
4121	*
4122	* @returns VBox status code.
4123	* @param pVM Pointer to the VM.
4124	* @param pVCPU Pointer to the VMCPU.
4125	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4126	* out-of-sync. Make sure to update the required fields
4127	* before using them.
4128	*
4129	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4130	* @remarks No-long-jump zone!!!
4131	*/
4132	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4133	{
4134	int rc = VERR_INTERNAL_ERROR_5;
4135	PVM pVM = pVCpu->CTX_SUFF(pVM);
4136
4137	/*
4138	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4139	*/
4140	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4141	{
4142	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4143	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4144	{
4145	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4146	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4147	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4148	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4149	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4150	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4151	}
4152
4153	#ifdef VBOX_WITH_REM
4154	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4155	{
4156	Assert(pVM->hm.s.vmx.pRealModeTSS);
4157	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4158	if ( pVCpu->hm.s.vmx.fWasInRealMode
4159	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4160	{
4161	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4162	in real-mode (e.g. OpenBSD 4.0) */
4163	REMFlushTBs(pVM);
4164	Log4(("Load: Switch to protected mode detected!\n"));
4165	pVCpu->hm.s.vmx.fWasInRealMode = false;
4166	}
4167	}
4168	#endif
4169	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4170	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4171	AssertRCReturn(rc, rc);
4172	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4173	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4174	AssertRCReturn(rc, rc);
4175	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4176	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4177	AssertRCReturn(rc, rc);
4178	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4179	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4180	AssertRCReturn(rc, rc);
4181	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4182	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4183	AssertRCReturn(rc, rc);
4184	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4185	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4186	AssertRCReturn(rc, rc);
4187
4188	#ifdef VBOX_STRICT
4189	/* Validate. */
4190	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4191	#endif
4192
4193	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4194	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4195	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4196	}
4197
4198	/*
4199	* Guest TR.
4200	*/
4201	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4202	{
4203	/*
4204	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4205	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4206	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4207	*/
4208	uint16_t u16Sel = 0;
4209	uint32_t u32Limit = 0;
4210	uint64_t u64Base = 0;
4211	uint32_t u32AccessRights = 0;
4212
4213	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4214	{
4215	u16Sel = pMixedCtx->tr.Sel;
4216	u32Limit = pMixedCtx->tr.u32Limit;
4217	u64Base = pMixedCtx->tr.u64Base;
4218	u32AccessRights = pMixedCtx->tr.Attr.u;
4219	}
4220	else
4221	{
4222	Assert(pVM->hm.s.vmx.pRealModeTSS);
4223	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4224
4225	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4226	RTGCPHYS GCPhys;
4227	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4228	AssertRCReturn(rc, rc);
4229
4230	X86DESCATTR DescAttr;
4231	DescAttr.u = 0;
4232	DescAttr.n.u1Present = 1;
4233	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4234
4235	u16Sel = 0;
4236	u32Limit = HM_VTX_TSS_SIZE;
4237	u64Base = GCPhys; /* in real-mode phys = virt. */
4238	u32AccessRights = DescAttr.u;
4239	}
4240
4241	/* Validate. */
4242	Assert(!(u16Sel & RT_BIT(2)));
4243	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4244	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4245	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4246	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4247	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4248	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4249	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4250	Assert( (u32Limit & 0xfff) == 0xfff
4251	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4252	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4253	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4254
4255	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4256	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4257	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4258	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4259
4260	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4261	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4262	}
4263
4264	/*
4265	* Guest GDTR.
4266	*/
4267	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4268	{
4269	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4270	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4271
4272	/* Validate. */
4273	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4274
4275	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4276	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4277	}
4278
4279	/*
4280	* Guest LDTR.
4281	*/
4282	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4283	{
4284	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4285	uint32_t u32Access = 0;
4286	if (!pMixedCtx->ldtr.Attr.u)
4287	u32Access = X86DESCATTR_UNUSABLE;
4288	else
4289	u32Access = pMixedCtx->ldtr.Attr.u;
4290
4291	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4292	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4293	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4294	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4295
4296	/* Validate. */
4297	if (!(u32Access & X86DESCATTR_UNUSABLE))
4298	{
4299	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4300	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4301	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4302	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4303	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4304	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4305	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4306	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4307	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4308	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4309	}
4310
4311	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4312	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4313	}
4314
4315	/*
4316	* Guest IDTR.
4317	*/
4318	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4319	{
4320	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4321	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4322
4323	/* Validate. */
4324	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4325
4326	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4327	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4328	}
4329
4330	return VINF_SUCCESS;
4331	}
4332
4333
4334	/**
4335	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4336	* areas. These MSRs will automatically be loaded to the host CPU on every
4337	* successful VM entry and stored from the host CPU on every successful VM exit.
4338	*
4339	* This also creates/updates MSR slots for the host MSRs. The actual host
4340	* MSR values are -not- updated here for performance reasons. See
4341	* hmR0VmxSaveHostMsrs().
4342	*
4343	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4344	*
4345	* @returns VBox status code.
4346	* @param pVCpu Pointer to the VMCPU.
4347	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4348	* out-of-sync. Make sure to update the required fields
4349	* before using them.
4350	*
4351	* @remarks No-long-jump zone!!!
4352	*/
4353	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4354	{
4355	AssertPtr(pVCpu);
4356	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4357
4358	/*
4359	* Shared MSRs that we use the auto-load/store MSR area in the VMCS.
4360	*/
4361	int rc = VINF_SUCCESS;
4362	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4363	{
4364	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4365	PVM pVM = pVCpu->CTX_SUFF(pVM);
4366	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4367	if (fSupportsLongMode)
4368	{
4369	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4370	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4371	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4372	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4373	}
4374
4375	# ifdef DEBUG
4376	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4377	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4378	Log4(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4379	# endif
4380	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4381	}
4382
4383	/*
4384	* Guest Sysenter MSRs.
4385	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4386	* VM-exits on WRMSRs for these MSRs.
4387	*/
4388	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4389	{
4390	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4391	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4392	}
4393
4394	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4395	{
4396	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4397	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4398	}
4399
4400	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4401	{
4402	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4403	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4404	}
4405
4406	return rc;
4407	}
4408
4409
4410	/**
4411	* Loads the guest activity state into the guest-state area in the VMCS.
4412	*
4413	* @returns VBox status code.
4414	* @param pVCpu Pointer to the VMCPU.
4415	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4416	* out-of-sync. Make sure to update the required fields
4417	* before using them.
4418	*
4419	* @remarks No-long-jump zone!!!
4420	*/
4421	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4422	{
4423	NOREF(pCtx);
4424	/** @todo See if we can make use of other states, e.g.
4425	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4426	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4427	{
4428	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4429	AssertRCReturn(rc, rc);
4430
4431	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4432	}
4433	return VINF_SUCCESS;
4434	}
4435
4436
4437	/**
4438	* Sets up the appropriate function to run guest code.
4439	*
4440	* @returns VBox status code.
4441	* @param pVCpu Pointer to the VMCPU.
4442	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4443	* out-of-sync. Make sure to update the required fields
4444	* before using them.
4445	*
4446	* @remarks No-long-jump zone!!!
4447	*/
4448	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4449	{
4450	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4451	{
4452	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4453	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4454	#endif
4455	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4456	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4457	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4458	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4459	{
4460	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4461	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4462	}
4463	#else
4464	/* 64-bit host or hybrid host. */
4465	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4466	#endif
4467	}
4468	else
4469	{
4470	/* Guest is not in long mode, use the 32-bit handler. */
4471	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4472	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4473	{
4474	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4475	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4476	}
4477	#else
4478	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4479	#endif
4480	}
4481	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4482	return VINF_SUCCESS;
4483	}
4484
4485
4486	/**
4487	* Wrapper for running the guest code in VT-x.
4488	*
4489	* @returns VBox strict status code.
4490	* @param pVM Pointer to the VM.
4491	* @param pVCpu Pointer to the VMCPU.
4492	* @param pCtx Pointer to the guest-CPU context.
4493	*
4494	* @remarks No-long-jump zone!!!
4495	*/
4496	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4497	{
4498	/*
4499	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4500	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4501	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4502	*/
4503	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4504	/** @todo Add stats for resume vs launch. */
4505	#ifdef VBOX_WITH_KERNEL_USING_XMM
4506	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4507	#else
4508	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4509	#endif
4510	}
4511
4512
4513	/**
4514	* Reports world-switch error and dumps some useful debug info.
4515	*
4516	* @param pVM Pointer to the VM.
4517	* @param pVCpu Pointer to the VMCPU.
4518	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4519	* @param pCtx Pointer to the guest-CPU context.
4520	* @param pVmxTransient Pointer to the VMX transient structure (only
4521	* exitReason updated).
4522	*/
4523	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4524	{
4525	Assert(pVM);
4526	Assert(pVCpu);
4527	Assert(pCtx);
4528	Assert(pVmxTransient);
4529	HMVMX_ASSERT_PREEMPT_SAFE();
4530
4531	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4532	switch (rcVMRun)
4533	{
4534	case VERR_VMX_INVALID_VMXON_PTR:
4535	AssertFailed();
4536	break;
4537	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4538	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4539	{
4540	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4541	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4542	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
4543	AssertRC(rc);
4544
4545	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4546	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4547	Cannot do it here as we may have been long preempted. */
4548
4549	#ifdef VBOX_STRICT
4550	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4551	pVmxTransient->uExitReason));
4552	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4553	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4554	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4555	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4556	else
4557	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4558	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4559	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4560
4561	/* VMX control bits. */
4562	uint32_t u32Val;
4563	uint64_t u64Val;
4564	HMVMXHCUINTREG uHCReg;
4565	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4566	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4567	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4568	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4569	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4570	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4571	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4572	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4574	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4575	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4576	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4577	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4578	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4579	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4580	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4581	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4582	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4583	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4584	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4585	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4586	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4587	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4588	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4589	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4590	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4591	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4592	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4593	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4594	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4595	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4596	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4597	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4598	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4599	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4600	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4601	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4602	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4603	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4604	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4605	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4606	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4607
4608	/* Guest bits. */
4609	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4610	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4611	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4612	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4613	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4614	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4615	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4616	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4617
4618	/* Host bits. */
4619	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4620	Log4(("Host CR0 %#RHr\n", uHCReg));
4621	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4622	Log4(("Host CR3 %#RHr\n", uHCReg));
4623	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4624	Log4(("Host CR4 %#RHr\n", uHCReg));
4625
4626	RTGDTR HostGdtr;
4627	PCX86DESCHC pDesc;
4628	ASMGetGDTR(&HostGdtr);
4629	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4630	Log4(("Host CS %#08x\n", u32Val));
4631	if (u32Val < HostGdtr.cbGdt)
4632	{
4633	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4634	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4635	}
4636
4637	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4638	Log4(("Host DS %#08x\n", u32Val));
4639	if (u32Val < HostGdtr.cbGdt)
4640	{
4641	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4642	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4643	}
4644
4645	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4646	Log4(("Host ES %#08x\n", u32Val));
4647	if (u32Val < HostGdtr.cbGdt)
4648	{
4649	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4650	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4651	}
4652
4653	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4654	Log4(("Host FS %#08x\n", u32Val));
4655	if (u32Val < HostGdtr.cbGdt)
4656	{
4657	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4658	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4659	}
4660
4661	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4662	Log4(("Host GS %#08x\n", u32Val));
4663	if (u32Val < HostGdtr.cbGdt)
4664	{
4665	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4666	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4667	}
4668
4669	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4670	Log4(("Host SS %#08x\n", u32Val));
4671	if (u32Val < HostGdtr.cbGdt)
4672	{
4673	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4674	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4675	}
4676
4677	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4678	Log4(("Host TR %#08x\n", u32Val));
4679	if (u32Val < HostGdtr.cbGdt)
4680	{
4681	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4682	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4683	}
4684
4685	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4686	Log4(("Host TR Base %#RHv\n", uHCReg));
4687	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4688	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4689	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4690	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4691	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4692	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4693	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4694	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4695	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4696	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4697	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4698	Log4(("Host RSP %#RHv\n", uHCReg));
4699	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4700	Log4(("Host RIP %#RHv\n", uHCReg));
4701	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4702	if (HMVMX_IS_64BIT_HOST_MODE())
4703	{
4704	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4705	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4706	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4707	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4708	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4709	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4710	}
4711	# endif
4712	#endif /* VBOX_STRICT */
4713	break;
4714	}
4715
4716	default:
4717	/* Impossible */
4718	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4719	break;
4720	}
4721	NOREF(pVM); NOREF(pCtx);
4722	}
4723
4724
4725	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4726	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4727	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4728	#endif
4729	#ifdef VBOX_STRICT
4730	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4731	{
4732	switch (idxField)
4733	{
4734	case VMX_VMCS_GUEST_RIP:
4735	case VMX_VMCS_GUEST_RSP:
4736	case VMX_VMCS_GUEST_SYSENTER_EIP:
4737	case VMX_VMCS_GUEST_SYSENTER_ESP:
4738	case VMX_VMCS_GUEST_GDTR_BASE:
4739	case VMX_VMCS_GUEST_IDTR_BASE:
4740	case VMX_VMCS_GUEST_CS_BASE:
4741	case VMX_VMCS_GUEST_DS_BASE:
4742	case VMX_VMCS_GUEST_ES_BASE:
4743	case VMX_VMCS_GUEST_FS_BASE:
4744	case VMX_VMCS_GUEST_GS_BASE:
4745	case VMX_VMCS_GUEST_SS_BASE:
4746	case VMX_VMCS_GUEST_LDTR_BASE:
4747	case VMX_VMCS_GUEST_TR_BASE:
4748	case VMX_VMCS_GUEST_CR3:
4749	return true;
4750	}
4751	return false;
4752	}
4753
4754	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4755	{
4756	switch (idxField)
4757	{
4758	/* Read-only fields. */
4759	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4760	return true;
4761	}
4762	/* Remaining readable fields should also be writable. */
4763	return hmR0VmxIsValidWriteField(idxField);
4764	}
4765	#endif /* VBOX_STRICT */
4766
4767
4768	/**
4769	* Executes the specified handler in 64-bit mode.
4770	*
4771	* @returns VBox status code.
4772	* @param pVM Pointer to the VM.
4773	* @param pVCpu Pointer to the VMCPU.
4774	* @param pCtx Pointer to the guest CPU context.
4775	* @param enmOp The operation to perform.
4776	* @param cbParam Number of parameters.
4777	* @param paParam Array of 32-bit parameters.
4778	*/
4779	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4780	uint32_t *paParam)
4781	{
4782	int rc, rc2;
4783	PHMGLOBALCPUINFO pCpu;
4784	RTHCPHYS HCPhysCpuPage;
4785	RTCCUINTREG uOldEflags;
4786
4787	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4788	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4789	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4790	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4791
4792	#ifdef VBOX_STRICT
4793	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4794	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4795
4796	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4797	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4798	#endif
4799
4800	/* Disable interrupts. */
4801	uOldEflags = ASMIntDisableFlags();
4802
4803	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4804	RTCPUID idHostCpu = RTMpCpuId();
4805	CPUMR0SetLApic(pVCpu, idHostCpu);
4806	#endif
4807
4808	pCpu = HMR0GetCurrentCpu();
4809	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4810
4811	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4812	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4813
4814	/* Leave VMX Root Mode. */
4815	VMXDisable();
4816
4817	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4818
4819	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4820	CPUMSetHyperEIP(pVCpu, enmOp);
4821	for (int i = (int)cbParam - 1; i >= 0; i--)
4822	CPUMPushHyper(pVCpu, paParam[i]);
4823
4824	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4825
4826	/* Call the switcher. */
4827	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4828	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4829
4830	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4831	/* Make sure the VMX instructions don't cause #UD faults. */
4832	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4833
4834	/* Re-enter VMX Root Mode */
4835	rc2 = VMXEnable(HCPhysCpuPage);
4836	if (RT_FAILURE(rc2))
4837	{
4838	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4839	ASMSetFlags(uOldEflags);
4840	return rc2;
4841	}
4842
4843	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4844	AssertRC(rc2);
4845	Assert(!(ASMGetFlags() & X86_EFL_IF));
4846	ASMSetFlags(uOldEflags);
4847	return rc;
4848	}
4849
4850
4851	/**
4852	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4853	* supporting 64-bit guests.
4854	*
4855	* @returns VBox status code.
4856	* @param fResume Whether to VMLAUNCH or VMRESUME.
4857	* @param pCtx Pointer to the guest-CPU context.
4858	* @param pCache Pointer to the VMCS cache.
4859	* @param pVM Pointer to the VM.
4860	* @param pVCpu Pointer to the VMCPU.
4861	*/
4862	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4863	{
4864	uint32_t aParam[6];
4865	PHMGLOBALCPUINFO pCpu = NULL;
4866	RTHCPHYS HCPhysCpuPage = 0;
4867	int rc = VERR_INTERNAL_ERROR_5;
4868
4869	pCpu = HMR0GetCurrentCpu();
4870	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4871
4872	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4873	pCache->uPos = 1;
4874	pCache->interPD = PGMGetInterPaeCR3(pVM);
4875	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4876	#endif
4877
4878	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4879	pCache->TestIn.HCPhysCpuPage = 0;
4880	pCache->TestIn.HCPhysVmcs = 0;
4881	pCache->TestIn.pCache = 0;
4882	pCache->TestOut.HCPhysVmcs = 0;
4883	pCache->TestOut.pCache = 0;
4884	pCache->TestOut.pCtx = 0;
4885	pCache->TestOut.eflags = 0;
4886	#endif
4887
4888	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4889	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4890	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4891	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4892	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4893	aParam[5] = 0;
4894
4895	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4896	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4897	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4898	#endif
4899	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4900
4901	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4902	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4903	Assert(pCtx->dr[4] == 10);
4904	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4905	#endif
4906
4907	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4908	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4909	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4910	pVCpu->hm.s.vmx.HCPhysVmcs));
4911	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4912	pCache->TestOut.HCPhysVmcs));
4913	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4914	pCache->TestOut.pCache));
4915	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4916	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4917	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4918	pCache->TestOut.pCtx));
4919	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4920	#endif
4921	return rc;
4922	}
4923
4924
4925	/**
4926	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4927	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4928	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4929	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4930	*
4931	* @returns VBox status code.
4932	* @param pVM Pointer to the VM.
4933	* @param pVCpu Pointer to the VMCPU.
4934	*/
4935	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4936	{
4937	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4938	{ \
4939	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4940	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4941	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4942	++cReadFields; \
4943	}
4944
4945	AssertPtr(pVM);
4946	AssertPtr(pVCpu);
4947	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4948	uint32_t cReadFields = 0;
4949
4950	/*
4951	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4952	* and serve to indicate exceptions to the rules.
4953	*/
4954
4955	/* Guest-natural selector base fields. */
4956	#if 0
4957	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4958	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4959	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4960	#endif
4961	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4962	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4963	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4964	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4965	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4966	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4967	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4968	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4969	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4970	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4971	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4972	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4973	#if 0
4974	/* Unused natural width guest-state fields. */
4975	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4976	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4977	#endif
4978	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4979	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4980
4981	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4982	#if 0
4983	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4984	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4985	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4986	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4987	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4988	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4989	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4990	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4991	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4992	#endif
4993
4994	/* Natural width guest-state fields. */
4995	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4996	#if 0
4997	/* Currently unused field. */
4998	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4999	#endif
5000
5001	if (pVM->hm.s.fNestedPaging)
5002	{
5003	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5004	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5005	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5006	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5007	}
5008	else
5009	{
5010	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5011	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5012	}
5013
5014	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5015	return VINF_SUCCESS;
5016	}
5017
5018
5019	/**
5020	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5021	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5022	* darwin, running 64-bit guests).
5023	*
5024	* @returns VBox status code.
5025	* @param pVCpu Pointer to the VMCPU.
5026	* @param idxField The VMCS field encoding.
5027	* @param u64Val 16, 32 or 64-bit value.
5028	*/
5029	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5030	{
5031	int rc;
5032	switch (idxField)
5033	{
5034	/*
5035	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5036	*/
5037	/* 64-bit Control fields. */
5038	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5039	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5040	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5041	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5042	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5043	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5044	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5045	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5046	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5047	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5048	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5049	case VMX_VMCS64_CTRL_EPTP_FULL:
5050	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5051	/* 64-bit Guest-state fields. */
5052	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5053	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5054	case VMX_VMCS64_GUEST_PAT_FULL:
5055	case VMX_VMCS64_GUEST_EFER_FULL:
5056	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5057	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5058	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5059	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5060	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5061	/* 64-bit Host-state fields. */
5062	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5063	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5064	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5065	{
5066	rc = VMXWriteVmcs32(idxField, u64Val);
5067	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5068	break;
5069	}
5070
5071	/*
5072	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5073	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5074	*/
5075	/* Natural-width Guest-state fields. */
5076	case VMX_VMCS_GUEST_CR3:
5077	case VMX_VMCS_GUEST_ES_BASE:
5078	case VMX_VMCS_GUEST_CS_BASE:
5079	case VMX_VMCS_GUEST_SS_BASE:
5080	case VMX_VMCS_GUEST_DS_BASE:
5081	case VMX_VMCS_GUEST_FS_BASE:
5082	case VMX_VMCS_GUEST_GS_BASE:
5083	case VMX_VMCS_GUEST_LDTR_BASE:
5084	case VMX_VMCS_GUEST_TR_BASE:
5085	case VMX_VMCS_GUEST_GDTR_BASE:
5086	case VMX_VMCS_GUEST_IDTR_BASE:
5087	case VMX_VMCS_GUEST_RSP:
5088	case VMX_VMCS_GUEST_RIP:
5089	case VMX_VMCS_GUEST_SYSENTER_ESP:
5090	case VMX_VMCS_GUEST_SYSENTER_EIP:
5091	{
5092	if (!(u64Val >> 32))
5093	{
5094	/* If this field is 64-bit, VT-x will zero out the top bits. */
5095	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5096	}
5097	else
5098	{
5099	/* Assert that only the 32->64 switcher case should ever come here. */
5100	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5101	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5102	}
5103	break;
5104	}
5105
5106	default:
5107	{
5108	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5109	rc = VERR_INVALID_PARAMETER;
5110	break;
5111	}
5112	}
5113	AssertRCReturn(rc, rc);
5114	return rc;
5115	}
5116
5117
5118	/**
5119	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5120	* hosts (except darwin) for 64-bit guests.
5121	*
5122	* @param pVCpu Pointer to the VMCPU.
5123	* @param idxField The VMCS field encoding.
5124	* @param u64Val 16, 32 or 64-bit value.
5125	*/
5126	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5127	{
5128	AssertPtr(pVCpu);
5129	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5130
5131	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5132	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5133
5134	/* Make sure there are no duplicates. */
5135	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5136	{
5137	if (pCache->Write.aField[i] == idxField)
5138	{
5139	pCache->Write.aFieldVal[i] = u64Val;
5140	return VINF_SUCCESS;
5141	}
5142	}
5143
5144	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5145	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5146	pCache->Write.cValidEntries++;
5147	return VINF_SUCCESS;
5148	}
5149
5150	/* Enable later when the assembly code uses these as callbacks. */
5151	#if 0
5152	/*
5153	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5154	*
5155	* @param pVCpu Pointer to the VMCPU.
5156	* @param pCache Pointer to the VMCS cache.
5157	*
5158	* @remarks No-long-jump zone!!!
5159	*/
5160	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5161	{
5162	AssertPtr(pCache);
5163	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5164	{
5165	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5166	AssertRC(rc);
5167	}
5168	pCache->Write.cValidEntries = 0;
5169	}
5170
5171
5172	/**
5173	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5174	*
5175	* @param pVCpu Pointer to the VMCPU.
5176	* @param pCache Pointer to the VMCS cache.
5177	*
5178	* @remarks No-long-jump zone!!!
5179	*/
5180	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5181	{
5182	AssertPtr(pCache);
5183	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5184	{
5185	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5186	AssertRC(rc);
5187	}
5188	}
5189	#endif
5190	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5191
5192
5193	/**
5194	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5195	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5196	* timer.
5197	*
5198	* @returns VBox status code.
5199	* @param pVCpu Pointer to the VMCPU.
5200	*
5201	* @remarks No-long-jump zone!!!
5202	*/
5203	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5204	{
5205	int rc = VERR_INTERNAL_ERROR_5;
5206	bool fOffsettedTsc = false;
5207	PVM pVM = pVCpu->CTX_SUFF(pVM);
5208	if (pVM->hm.s.vmx.fUsePreemptTimer)
5209	{
5210	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5211
5212	/* Make sure the returned values have sane upper and lower boundaries. */
5213	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5214	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5215	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5216	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5217
5218	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5219	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5220	}
5221	else
5222	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5223
5224	if (fOffsettedTsc)
5225	{
5226	uint64_t u64CurTSC = ASMReadTSC();
5227	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5228	{
5229	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5230	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5231
5232	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5233	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5234	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5235	}
5236	else
5237	{
5238	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5239	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5240	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5241	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5242	}
5243	}
5244	else
5245	{
5246	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5247	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5248	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5249	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5250	}
5251	}
5252
5253
5254	/**
5255	* Determines if an exception is a contributory exception. Contributory
5256	* exceptions are ones which can cause double-faults. Page-fault is
5257	* intentionally not included here as it's a conditional contributory exception.
5258	*
5259	* @returns true if the exception is contributory, false otherwise.
5260	* @param uVector The exception vector.
5261	*/
5262	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5263	{
5264	switch (uVector)
5265	{
5266	case X86_XCPT_GP:
5267	case X86_XCPT_SS:
5268	case X86_XCPT_NP:
5269	case X86_XCPT_TS:
5270	case X86_XCPT_DE:
5271	return true;
5272	default:
5273	break;
5274	}
5275	return false;
5276	}
5277
5278
5279	/**
5280	* Sets an event as a pending event to be injected into the guest.
5281	*
5282	* @param pVCpu Pointer to the VMCPU.
5283	* @param u32IntInfo The VM-entry interruption-information field.
5284	* @param cbInstr The VM-entry instruction length in bytes (for software
5285	* interrupts, exceptions and privileged software
5286	* exceptions).
5287	* @param u32ErrCode The VM-entry exception error code.
5288	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5289	* page-fault.
5290	*
5291	* @remarks Statistics counter assumes this is a guest event being injected or
5292	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5293	* always incremented.
5294	*/
5295	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5296	RTGCUINTPTR GCPtrFaultAddress)
5297	{
5298	Assert(!pVCpu->hm.s.Event.fPending);
5299	pVCpu->hm.s.Event.fPending = true;
5300	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5301	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5302	pVCpu->hm.s.Event.cbInstr = cbInstr;
5303	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5304
5305	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5306	}
5307
5308
5309	/**
5310	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5311	*
5312	* @param pVCpu Pointer to the VMCPU.
5313	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5314	* out-of-sync. Make sure to update the required fields
5315	* before using them.
5316	*/
5317	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5318	{
5319	NOREF(pMixedCtx);
5320	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5321	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5322	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5323	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5324	}
5325
5326
5327	/**
5328	* Handle a condition that occurred while delivering an event through the guest
5329	* IDT.
5330	*
5331	* @returns VBox status code (informational error codes included).
5332	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5333	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5334	* continue execution of the guest which will delivery the #DF.
5335	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5336	*
5337	* @param pVCpu Pointer to the VMCPU.
5338	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5339	* out-of-sync. Make sure to update the required fields
5340	* before using them.
5341	* @param pVmxTransient Pointer to the VMX transient structure.
5342	*
5343	* @remarks No-long-jump zone!!!
5344	*/
5345	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5346	{
5347	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5348	AssertRCReturn(rc, rc);
5349	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5350	{
5351	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5352	AssertRCReturn(rc, rc);
5353
5354	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5355	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5356	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5357
5358	typedef enum
5359	{
5360	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5361	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5362	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5363	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5364	} VMXREFLECTXCPT;
5365
5366	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5367	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5368	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5369	{
5370	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5371	{
5372	enmReflect = VMXREFLECTXCPT_XCPT;
5373	#ifdef VBOX_STRICT
5374	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5375	&& uExitVector == X86_XCPT_PF)
5376	{
5377	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5378	}
5379	#endif
5380	if ( uExitVector == X86_XCPT_PF
5381	&& uIdtVector == X86_XCPT_PF)
5382	{
5383	pVmxTransient->fVectoringPF = true;
5384	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5385	}
5386	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5387	&& hmR0VmxIsContributoryXcpt(uExitVector)
5388	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5389	\|\| uIdtVector == X86_XCPT_PF))
5390	{
5391	enmReflect = VMXREFLECTXCPT_DF;
5392	}
5393	else if (uIdtVector == X86_XCPT_DF)
5394	enmReflect = VMXREFLECTXCPT_TF;
5395	}
5396	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5397	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5398	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5399	{
5400	/*
5401	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5402	* (whatever they are) as they reoccur when restarting the instruction.
5403	*/
5404	enmReflect = VMXREFLECTXCPT_XCPT;
5405	}
5406	}
5407	else
5408	{
5409	/*
5410	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5411	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5412	* original exception to the guest after handling the VM-exit.
5413	*/
5414	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5415	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5416	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5417	{
5418	enmReflect = VMXREFLECTXCPT_XCPT;
5419	}
5420	}
5421
5422	switch (enmReflect)
5423	{
5424	case VMXREFLECTXCPT_XCPT:
5425	{
5426	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5427	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5428	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5429
5430	uint32_t u32ErrCode = 0;
5431	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5432	{
5433	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5434	AssertRCReturn(rc, rc);
5435	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5436	}
5437
5438	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5439	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5440	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5441	rc = VINF_SUCCESS;
5442	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5443	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5444
5445	break;
5446	}
5447
5448	case VMXREFLECTXCPT_DF:
5449	{
5450	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5451	rc = VINF_HM_DOUBLE_FAULT;
5452	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5453	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5454
5455	break;
5456	}
5457
5458	case VMXREFLECTXCPT_TF:
5459	{
5460	rc = VINF_EM_RESET;
5461	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5462	uExitVector));
5463	break;
5464	}
5465
5466	default:
5467	Assert(rc == VINF_SUCCESS);
5468	break;
5469	}
5470	}
5471	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5472	return rc;
5473	}
5474
5475
5476	/**
5477	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5478	*
5479	* @returns VBox status code.
5480	* @param pVCpu Pointer to the VMCPU.
5481	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5482	* out-of-sync. Make sure to update the required fields
5483	* before using them.
5484	*
5485	* @remarks No-long-jump zone!!!
5486	*/
5487	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5488	{
5489	NOREF(pMixedCtx);
5490
5491	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5492	{
5493	uint32_t uVal = 0;
5494	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5495	AssertRCReturn(rc, rc);
5496
5497	uint32_t uShadow = 0;
5498	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5499	AssertRCReturn(rc, rc);
5500
5501	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5502	CPUMSetGuestCR0(pVCpu, uVal);
5503	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5504	}
5505	return VINF_SUCCESS;
5506	}
5507
5508
5509	/**
5510	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5511	*
5512	* @returns VBox status code.
5513	* @param pVCpu Pointer to the VMCPU.
5514	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5515	* out-of-sync. Make sure to update the required fields
5516	* before using them.
5517	*
5518	* @remarks No-long-jump zone!!!
5519	*/
5520	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5521	{
5522	NOREF(pMixedCtx);
5523
5524	int rc = VINF_SUCCESS;
5525	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5526	{
5527	uint32_t uVal = 0;
5528	uint32_t uShadow = 0;
5529	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5530	AssertRCReturn(rc, rc);
5531	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5532	AssertRCReturn(rc, rc);
5533
5534	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5535	CPUMSetGuestCR4(pVCpu, uVal);
5536	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5537	}
5538	return rc;
5539	}
5540
5541
5542	/**
5543	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5544	*
5545	* @returns VBox status code.
5546	* @param pVCpu Pointer to the VMCPU.
5547	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5548	* out-of-sync. Make sure to update the required fields
5549	* before using them.
5550	*
5551	* @remarks No-long-jump zone!!!
5552	*/
5553	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5554	{
5555	int rc = VINF_SUCCESS;
5556	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5557	{
5558	uint64_t u64Val = 0;
5559	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5560	AssertRCReturn(rc, rc);
5561
5562	pMixedCtx->rip = u64Val;
5563	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5564	}
5565	return rc;
5566	}
5567
5568
5569	/**
5570	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5571	*
5572	* @returns VBox status code.
5573	* @param pVCpu Pointer to the VMCPU.
5574	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5575	* out-of-sync. Make sure to update the required fields
5576	* before using them.
5577	*
5578	* @remarks No-long-jump zone!!!
5579	*/
5580	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5581	{
5582	int rc = VINF_SUCCESS;
5583	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5584	{
5585	uint64_t u64Val = 0;
5586	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5587	AssertRCReturn(rc, rc);
5588
5589	pMixedCtx->rsp = u64Val;
5590	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5591	}
5592	return rc;
5593	}
5594
5595
5596	/**
5597	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5598	*
5599	* @returns VBox status code.
5600	* @param pVCpu Pointer to the VMCPU.
5601	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5602	* out-of-sync. Make sure to update the required fields
5603	* before using them.
5604	*
5605	* @remarks No-long-jump zone!!!
5606	*/
5607	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5608	{
5609	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5610	{
5611	uint32_t uVal = 0;
5612	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5613	AssertRCReturn(rc, rc);
5614
5615	pMixedCtx->eflags.u32 = uVal;
5616	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5617	{
5618	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5619	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5620
5621	pMixedCtx->eflags.Bits.u1VM = 0;
5622	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5623	}
5624
5625	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5626	}
5627	return VINF_SUCCESS;
5628	}
5629
5630
5631	/**
5632	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5633	* guest-CPU context.
5634	*/
5635	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5636	{
5637	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5638	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5639	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5640	return rc;
5641	}
5642
5643
5644	/**
5645	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5646	* from the guest-state area in the VMCS.
5647	*
5648	* @param pVCpu Pointer to the VMCPU.
5649	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5650	* out-of-sync. Make sure to update the required fields
5651	* before using them.
5652	*
5653	* @remarks No-long-jump zone!!!
5654	*/
5655	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5656	{
5657	uint32_t uIntrState = 0;
5658	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5659	AssertRC(rc);
5660
5661	if (!uIntrState)
5662	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5663	else
5664	{
5665	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5666	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5667	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5668	AssertRC(rc);
5669	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5670	AssertRC(rc);
5671
5672	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5673	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5674	}
5675	}
5676
5677
5678	/**
5679	* Saves the guest's activity state.
5680	*
5681	* @returns VBox status code.
5682	* @param pVCpu Pointer to the VMCPU.
5683	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5684	* out-of-sync. Make sure to update the required fields
5685	* before using them.
5686	*
5687	* @remarks No-long-jump zone!!!
5688	*/
5689	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5690	{
5691	NOREF(pMixedCtx);
5692	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5693	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5694	return VINF_SUCCESS;
5695	}
5696
5697
5698	/**
5699	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5700	* the current VMCS into the guest-CPU context.
5701	*
5702	* @returns VBox status code.
5703	* @param pVCpu Pointer to the VMCPU.
5704	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5705	* out-of-sync. Make sure to update the required fields
5706	* before using them.
5707	*
5708	* @remarks No-long-jump zone!!!
5709	*/
5710	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5711	{
5712	int rc = VINF_SUCCESS;
5713	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5714	{
5715	uint32_t u32Val = 0;
5716	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5717	pMixedCtx->SysEnter.cs = u32Val;
5718	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5719	}
5720
5721	uint64_t u64Val = 0;
5722	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5723	{
5724	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5725	pMixedCtx->SysEnter.eip = u64Val;
5726	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5727	}
5728	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5729	{
5730	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5731	pMixedCtx->SysEnter.esp = u64Val;
5732	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5733	}
5734	return rc;
5735	}
5736
5737
5738	/**
5739	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5740	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5741	* and TSC_AUX.
5742	*
5743	* @returns VBox status code.
5744	* @param pVCpu Pointer to the VMCPU.
5745	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5746	* out-of-sync. Make sure to update the required fields
5747	* before using them.
5748	*
5749	* @remarks No-long-jump zone!!!
5750	*/
5751	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5752	{
5753	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5754	return VINF_SUCCESS;
5755
5756	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5757	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", pVCpu->hm.s.vmx.cMsrs));
5758	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
5759	{
5760	switch (pMsr->u32Msr)
5761	{
5762	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5763	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5764	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5765	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5766	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5767	default:
5768	{
5769	AssertFailed();
5770	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5771	}
5772	}
5773	}
5774
5775	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5776	return VINF_SUCCESS;
5777	}
5778
5779
5780	/**
5781	* Saves the guest control registers from the current VMCS into the guest-CPU
5782	* context.
5783	*
5784	* @returns VBox status code.
5785	* @param pVCpu Pointer to the VMCPU.
5786	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5787	* out-of-sync. Make sure to update the required fields
5788	* before using them.
5789	*
5790	* @remarks No-long-jump zone!!!
5791	*/
5792	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5793	{
5794	/* Guest CR0. Guest FPU. */
5795	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5796	AssertRCReturn(rc, rc);
5797
5798	/* Guest CR4. */
5799	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5800	AssertRCReturn(rc, rc);
5801
5802	/* Guest CR2 - updated always during the world-switch or in #PF. */
5803	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5804	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5805	{
5806	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5807	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5808
5809	PVM pVM = pVCpu->CTX_SUFF(pVM);
5810	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5811	\|\| ( pVM->hm.s.fNestedPaging
5812	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5813	{
5814	uint64_t u64Val = 0;
5815	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5816	if (pMixedCtx->cr3 != u64Val)
5817	{
5818	CPUMSetGuestCR3(pVCpu, u64Val);
5819	if (VMMRZCallRing3IsEnabled(pVCpu))
5820	{
5821	PGMUpdateCR3(pVCpu, u64Val);
5822	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5823	}
5824	else
5825	{
5826	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5827	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5828	}
5829	}
5830
5831	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5832	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5833	{
5834	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5835	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5836	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5837	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5838
5839	if (VMMRZCallRing3IsEnabled(pVCpu))
5840	{
5841	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5842	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5843	}
5844	else
5845	{
5846	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5847	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5848	}
5849	}
5850	}
5851
5852	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5853	}
5854
5855	/*
5856	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5857	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5858	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5859	*
5860	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5861	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5862	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5863	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5864	*
5865	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5866	*/
5867	if (VMMRZCallRing3IsEnabled(pVCpu))
5868	{
5869	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5870	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5871
5872	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5873	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5874
5875	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5876	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5877	}
5878
5879	return rc;
5880	}
5881
5882
5883	/**
5884	* Reads a guest segment register from the current VMCS into the guest-CPU
5885	* context.
5886	*
5887	* @returns VBox status code.
5888	* @param pVCpu Pointer to the VMCPU.
5889	* @param idxSel Index of the selector in the VMCS.
5890	* @param idxLimit Index of the segment limit in the VMCS.
5891	* @param idxBase Index of the segment base in the VMCS.
5892	* @param idxAccess Index of the access rights of the segment in the VMCS.
5893	* @param pSelReg Pointer to the segment selector.
5894	*
5895	* @remarks No-long-jump zone!!!
5896	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5897	* macro as that takes care of whether to read from the VMCS cache or
5898	* not.
5899	*/
5900	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5901	PCPUMSELREG pSelReg)
5902	{
5903	NOREF(pVCpu);
5904
5905	uint32_t u32Val = 0;
5906	int rc = VMXReadVmcs32(idxSel, &u32Val);
5907	AssertRCReturn(rc, rc);
5908	pSelReg->Sel = (uint16_t)u32Val;
5909	pSelReg->ValidSel = (uint16_t)u32Val;
5910	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5911
5912	rc = VMXReadVmcs32(idxLimit, &u32Val);
5913	AssertRCReturn(rc, rc);
5914	pSelReg->u32Limit = u32Val;
5915
5916	uint64_t u64Val = 0;
5917	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5918	AssertRCReturn(rc, rc);
5919	pSelReg->u64Base = u64Val;
5920
5921	rc = VMXReadVmcs32(idxAccess, &u32Val);
5922	AssertRCReturn(rc, rc);
5923	pSelReg->Attr.u = u32Val;
5924
5925	/*
5926	* If VT-x marks the segment as unusable, most other bits remain undefined:
5927	* - For CS the L, D and G bits have meaning.
5928	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5929	* - For the remaining data segments no bits are defined.
5930	*
5931	* The present bit and the unusable bit has been observed to be set at the
5932	* same time (the selector was supposed to invalid as we started executing
5933	* a V8086 interrupt in ring-0).
5934	*
5935	* What should be important for the rest of the VBox code that the P bit is
5936	* cleared. Some of the other VBox code recognizes the unusable bit, but
5937	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5938	* safe side here, we'll strip off P and other bits we don't care about. If
5939	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5940	*
5941	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5942	*/
5943	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5944	{
5945	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5946
5947	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5948	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5949	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5950
5951	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5952	#ifdef DEBUG_bird
5953	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5954	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5955	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5956	#endif
5957	}
5958	return VINF_SUCCESS;
5959	}
5960
5961
5962	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5963	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5964	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5965	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5966	#else
5967	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5968	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5969	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5970	#endif
5971
5972
5973	/**
5974	* Saves the guest segment registers from the current VMCS into the guest-CPU
5975	* context.
5976	*
5977	* @returns VBox status code.
5978	* @param pVCpu Pointer to the VMCPU.
5979	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5980	* out-of-sync. Make sure to update the required fields
5981	* before using them.
5982	*
5983	* @remarks No-long-jump zone!!!
5984	*/
5985	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5986	{
5987	/* Guest segment registers. */
5988	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5989	{
5990	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5991	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5992	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5993	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5994	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5995	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5996	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5997
5998	/* Restore segment attributes for real-on-v86 mode hack. */
5999	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6000	{
6001	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6002	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6003	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6004	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6005	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6006	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6007	}
6008	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
6009	}
6010
6011	return VINF_SUCCESS;
6012	}
6013
6014
6015	/**
6016	* Saves the guest descriptor table registers and task register from the current
6017	* VMCS into the guest-CPU context.
6018	*
6019	* @returns VBox status code.
6020	* @param pVCpu Pointer to the VMCPU.
6021	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6022	* out-of-sync. Make sure to update the required fields
6023	* before using them.
6024	*
6025	* @remarks No-long-jump zone!!!
6026	*/
6027	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6028	{
6029	int rc = VINF_SUCCESS;
6030
6031	/* Guest LDTR. */
6032	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
6033	{
6034	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6035	AssertRCReturn(rc, rc);
6036	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
6037	}
6038
6039	/* Guest GDTR. */
6040	uint64_t u64Val = 0;
6041	uint32_t u32Val = 0;
6042	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6043	{
6044	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6045	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6046	pMixedCtx->gdtr.pGdt = u64Val;
6047	pMixedCtx->gdtr.cbGdt = u32Val;
6048	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
6049	}
6050
6051	/* Guest IDTR. */
6052	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6053	{
6054	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6055	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6056	pMixedCtx->idtr.pIdt = u64Val;
6057	pMixedCtx->idtr.cbIdt = u32Val;
6058	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
6059	}
6060
6061	/* Guest TR. */
6062	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6063	{
6064	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6065	AssertRCReturn(rc, rc);
6066
6067	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6068	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6069	{
6070	rc = VMXLOCAL_READ_SEG(TR, tr);
6071	AssertRCReturn(rc, rc);
6072	}
6073	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
6074	}
6075	return rc;
6076	}
6077
6078	#undef VMXLOCAL_READ_SEG
6079
6080
6081	/**
6082	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6083	* context.
6084	*
6085	* @returns VBox status code.
6086	* @param pVCpu Pointer to the VMCPU.
6087	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6088	* out-of-sync. Make sure to update the required fields
6089	* before using them.
6090	*
6091	* @remarks No-long-jump zone!!!
6092	*/
6093	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6094	{
6095	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6096	{
6097	if (!pVCpu->hm.s.fUsingHyperDR7)
6098	{
6099	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6100	uint32_t u32Val;
6101	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6102	pMixedCtx->dr[7] = u32Val;
6103	}
6104
6105	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
6106	}
6107	return VINF_SUCCESS;
6108	}
6109
6110
6111	/**
6112	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6113	*
6114	* @returns VBox status code.
6115	* @param pVCpu Pointer to the VMCPU.
6116	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6117	* out-of-sync. Make sure to update the required fields
6118	* before using them.
6119	*
6120	* @remarks No-long-jump zone!!!
6121	*/
6122	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6123	{
6124	NOREF(pMixedCtx);
6125
6126	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6127	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
6128	return VINF_SUCCESS;
6129	}
6130
6131
6132	/**
6133	* Saves the entire guest state from the currently active VMCS into the
6134	* guest-CPU context. This essentially VMREADs all guest-data.
6135	*
6136	* @returns VBox status code.
6137	* @param pVCpu Pointer to the VMCPU.
6138	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6139	* out-of-sync. Make sure to update the required fields
6140	* before using them.
6141	*/
6142	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6143	{
6144	Assert(pVCpu);
6145	Assert(pMixedCtx);
6146
6147	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6148	return VINF_SUCCESS;
6149
6150	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6151	again on the ring-3 callback path, there is no real need to. */
6152	if (VMMRZCallRing3IsEnabled(pVCpu))
6153	VMMR0LogFlushDisable(pVCpu);
6154	else
6155	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6156	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6157
6158	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6159	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6160
6161	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6162	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6163
6164	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6165	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6166
6167	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6168	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6169
6170	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6171	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6172
6173	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6174	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6175
6176	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6177	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6178
6179	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6180	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6181
6182	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6183	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6184
6185	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6186	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6187
6188	if (VMMRZCallRing3IsEnabled(pVCpu))
6189	VMMR0LogFlushEnable(pVCpu);
6190
6191	return rc;
6192	}
6193
6194
6195	/**
6196	* Check per-VM and per-VCPU force flag actions that require us to go back to
6197	* ring-3 for one reason or another.
6198	*
6199	* @returns VBox status code (information status code included).
6200	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6201	* ring-3.
6202	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6203	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6204	* interrupts)
6205	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6206	* all EMTs to be in ring-3.
6207	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6208	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6209	* to the EM loop.
6210	*
6211	* @param pVM Pointer to the VM.
6212	* @param pVCpu Pointer to the VMCPU.
6213	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6214	* out-of-sync. Make sure to update the required fields
6215	* before using them.
6216	*/
6217	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6218	{
6219	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6220
6221	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6222	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6223	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6224	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6225	{
6226	/* We need the control registers now, make sure the guest-CPU context is updated. */
6227	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6228	AssertRCReturn(rc3, rc3);
6229
6230	/* Pending HM CR3 sync. */
6231	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6232	{
6233	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6234	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6235	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6236	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6237	}
6238
6239	/* Pending HM PAE PDPEs. */
6240	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6241	{
6242	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6243	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6244	}
6245
6246	/* Pending PGM C3 sync. */
6247	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6248	{
6249	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6250	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6251	if (rc2 != VINF_SUCCESS)
6252	{
6253	AssertRC(rc2);
6254	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6255	return rc2;
6256	}
6257	}
6258
6259	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6260	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6261	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6262	{
6263	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6264	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6265	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6266	return rc2;
6267	}
6268
6269	/* Pending VM request packets, such as hardware interrupts. */
6270	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6271	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6272	{
6273	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6274	return VINF_EM_PENDING_REQUEST;
6275	}
6276
6277	/* Pending PGM pool flushes. */
6278	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6279	{
6280	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6281	return VINF_PGM_POOL_FLUSH_PENDING;
6282	}
6283
6284	/* Pending DMA requests. */
6285	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6286	{
6287	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6288	return VINF_EM_RAW_TO_R3;
6289	}
6290	}
6291
6292	return VINF_SUCCESS;
6293	}
6294
6295
6296	/**
6297	* Converts any TRPM trap into a pending HM event. This is typically used when
6298	* entering from ring-3 (not longjmp returns).
6299	*
6300	* @param pVCpu Pointer to the VMCPU.
6301	*/
6302	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6303	{
6304	Assert(TRPMHasTrap(pVCpu));
6305	Assert(!pVCpu->hm.s.Event.fPending);
6306
6307	uint8_t uVector;
6308	TRPMEVENT enmTrpmEvent;
6309	RTGCUINT uErrCode;
6310	RTGCUINTPTR GCPtrFaultAddress;
6311	uint8_t cbInstr;
6312
6313	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6314	AssertRC(rc);
6315
6316	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6317	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6318	if (enmTrpmEvent == TRPM_TRAP)
6319	{
6320	switch (uVector)
6321	{
6322	case X86_XCPT_BP:
6323	case X86_XCPT_OF:
6324	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6325	break;
6326
6327	case X86_XCPT_PF:
6328	case X86_XCPT_DF:
6329	case X86_XCPT_TS:
6330	case X86_XCPT_NP:
6331	case X86_XCPT_SS:
6332	case X86_XCPT_GP:
6333	case X86_XCPT_AC:
6334	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6335	/* no break! */
6336	default:
6337	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6338	break;
6339	}
6340	}
6341	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6342	{
6343	if (uVector == X86_XCPT_NMI)
6344	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6345	else
6346	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6347	}
6348	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6349	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6350	else
6351	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6352
6353	rc = TRPMResetTrap(pVCpu);
6354	AssertRC(rc);
6355	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6356	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6357
6358	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6359	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6360	}
6361
6362
6363	/**
6364	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6365	* VT-x to execute any instruction.
6366	*
6367	* @param pvCpu Pointer to the VMCPU.
6368	*/
6369	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6370	{
6371	Assert(pVCpu->hm.s.Event.fPending);
6372
6373	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6374	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6375	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6376	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6377
6378	/* If a trap was already pending, we did something wrong! */
6379	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6380
6381	TRPMEVENT enmTrapType;
6382	switch (uVectorType)
6383	{
6384	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6385	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6386	enmTrapType = TRPM_HARDWARE_INT;
6387	break;
6388
6389	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6390	enmTrapType = TRPM_SOFTWARE_INT;
6391	break;
6392
6393	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6394	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6395	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6396	enmTrapType = TRPM_TRAP;
6397	break;
6398
6399	default:
6400	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6401	enmTrapType = TRPM_32BIT_HACK;
6402	break;
6403	}
6404
6405	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6406
6407	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6408	AssertRC(rc);
6409
6410	if (fErrorCodeValid)
6411	TRPMSetErrorCode(pVCpu, uErrorCode);
6412
6413	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6414	&& uVector == X86_XCPT_PF)
6415	{
6416	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6417	}
6418	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6419	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6420	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6421	{
6422	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6423	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6424	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6425	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6426	}
6427	pVCpu->hm.s.Event.fPending = false;
6428	}
6429
6430
6431	/**
6432	* Does the necessary state syncing before returning to ring-3 for any reason
6433	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6434	*
6435	* @returns VBox status code.
6436	* @param pVM Pointer to the VM.
6437	* @param pVCpu Pointer to the VMCPU.
6438	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6439	* be out-of-sync. Make sure to update the required
6440	* fields before using them.
6441	* @param fSaveGuestState Whether to save the guest state or not.
6442	*
6443	* @remarks If you modify code here, make sure to check whether
6444	* hmR0VmxCallRing3Callback() needs to be updated too.
6445	* @remarks No-long-jmp zone!!!
6446	*/
6447	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6448	{
6449	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6450	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6451
6452	RTCPUID idCpu = RTMpCpuId();
6453	Log4Func(("HostCpuId=%u\n", idCpu));
6454
6455	/* Save the guest state if necessary. */
6456	if ( fSaveGuestState
6457	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6458	{
6459	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6460	AssertRCReturn(rc, rc);
6461	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6462	}
6463
6464	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6465	if (CPUMIsGuestFPUStateActive(pVCpu))
6466	{
6467	/* We shouldn't reload CR0 without saving it first. */
6468	if (!fSaveGuestState)
6469	{
6470	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6471	AssertRCReturn(rc, rc);
6472	}
6473	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6474	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6475	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6476	}
6477
6478	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6479	#ifdef VBOX_STRICT
6480	if (CPUMIsHyperDebugStateActive(pVCpu))
6481	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6482	#endif
6483	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6484	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6485	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6486	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6487
6488	#if HC_ARCH_BITS == 64
6489	/* Restore host-state bits that VT-x only restores partially. */
6490	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6491	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6492	{
6493	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6494	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6495	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6496	}
6497	#endif
6498
6499	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6500	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6501	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6502	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6503	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6504	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6505	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6506	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6507
6508	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6509
6510	/** @todo This kinda defeats the purpose of having preemption hooks.
6511	* The problem is, deregistering the hooks should be moved to a place that
6512	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6513	* context.
6514	*/
6515	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6516	{
6517	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6518	AssertRCReturn(rc, rc);
6519
6520	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6521	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6522	}
6523	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6524	NOREF(idCpu);
6525
6526	return VINF_SUCCESS;
6527	}
6528
6529
6530	/**
6531	* Leaves the VT-x session.
6532	*
6533	* @returns VBox status code.
6534	* @param pVM Pointer to the VM.
6535	* @param pVCpu Pointer to the VMCPU.
6536	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6537	* out-of-sync. Make sure to update the required fields
6538	* before using them.
6539	*
6540	* @remarks No-long-jmp zone!!!
6541	*/
6542	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6543	{
6544	HM_DISABLE_PREEMPT_IF_NEEDED();
6545	HMVMX_ASSERT_CPU_SAFE();
6546	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6547	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6548
6549	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6550	and done this from the VMXR0ThreadCtxCallback(). */
6551	if (!pVCpu->hm.s.fLeaveDone)
6552	{
6553	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6554	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6555	pVCpu->hm.s.fLeaveDone = true;
6556	}
6557
6558	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6559	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6560	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6561	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6562	VMMR0ThreadCtxHooksDeregister(pVCpu);
6563
6564	/* Leave HM context. This takes care of local init (term). */
6565	int rc = HMR0LeaveCpu(pVCpu);
6566
6567	HM_RESTORE_PREEMPT_IF_NEEDED();
6568
6569	return rc;
6570	}
6571
6572
6573	/**
6574	* Does the necessary state syncing before doing a longjmp to ring-3.
6575	*
6576	* @returns VBox status code.
6577	* @param pVM Pointer to the VM.
6578	* @param pVCpu Pointer to the VMCPU.
6579	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6580	* out-of-sync. Make sure to update the required fields
6581	* before using them.
6582	*
6583	* @remarks No-long-jmp zone!!!
6584	*/
6585	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6586	{
6587	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6588	}
6589
6590
6591	/**
6592	* Take necessary actions before going back to ring-3.
6593	*
6594	* An action requires us to go back to ring-3. This function does the necessary
6595	* steps before we can safely return to ring-3. This is not the same as longjmps
6596	* to ring-3, this is voluntary and prepares the guest so it may continue
6597	* executing outside HM (recompiler/IEM).
6598	*
6599	* @returns VBox status code.
6600	* @param pVM Pointer to the VM.
6601	* @param pVCpu Pointer to the VMCPU.
6602	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6603	* out-of-sync. Make sure to update the required fields
6604	* before using them.
6605	* @param rcExit The reason for exiting to ring-3. Can be
6606	* VINF_VMM_UNKNOWN_RING3_CALL.
6607	*/
6608	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6609	{
6610	Assert(pVM);
6611	Assert(pVCpu);
6612	Assert(pMixedCtx);
6613	HMVMX_ASSERT_PREEMPT_SAFE();
6614
6615	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6616	{
6617	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6618	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6619	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6620	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6621	}
6622
6623	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6624	VMMRZCallRing3Disable(pVCpu);
6625	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6626
6627	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6628	if (pVCpu->hm.s.Event.fPending)
6629	{
6630	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6631	Assert(!pVCpu->hm.s.Event.fPending);
6632	}
6633
6634	/* Save guest state and restore host state bits. */
6635	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6636	AssertRCReturn(rc, rc);
6637	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6638
6639	/* Sync recompiler state. */
6640	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6641	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6642	\| CPUM_CHANGED_LDTR
6643	\| CPUM_CHANGED_GDTR
6644	\| CPUM_CHANGED_IDTR
6645	\| CPUM_CHANGED_TR
6646	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6647	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6648	if ( pVM->hm.s.fNestedPaging
6649	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6650	{
6651	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6652	}
6653
6654	Assert(!pVCpu->hm.s.fClearTrapFlag);
6655
6656	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6657	if (rcExit != VINF_EM_RAW_INTERRUPT)
6658	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6659
6660	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6661
6662	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6663	VMMRZCallRing3RemoveNotification(pVCpu);
6664	VMMRZCallRing3Enable(pVCpu);
6665
6666	return rc;
6667	}
6668
6669
6670	/**
6671	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6672	* longjump to ring-3 and possibly get preempted.
6673	*
6674	* @returns VBox status code.
6675	* @param pVCpu Pointer to the VMCPU.
6676	* @param enmOperation The operation causing the ring-3 longjump.
6677	* @param pvUser Opaque pointer to the guest-CPU context. The data
6678	* may be out-of-sync. Make sure to update the required
6679	* fields before using them.
6680	* @remarks If you modify code here, make sure to check whether
6681	* hmR0VmxLeave() needs to be updated too.
6682	*/
6683	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6684	{
6685	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6686	{
6687	VMMRZCallRing3RemoveNotification(pVCpu);
6688	HM_DISABLE_PREEMPT_IF_NEEDED();
6689
6690	/* If anything here asserts or fails, good luck. */
6691	if (CPUMIsGuestFPUStateActive(pVCpu))
6692	CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6693
6694	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6695
6696	#if HC_ARCH_BITS == 64
6697	/* Restore host-state bits that VT-x only restores partially. */
6698	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6699	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6700	{
6701	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6702	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6703	}
6704	#endif
6705	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6706	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6707	{
6708	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6709	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6710	}
6711
6712	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6713	VMMR0ThreadCtxHooksDeregister(pVCpu);
6714
6715	HMR0LeaveCpu(pVCpu);
6716	HM_RESTORE_PREEMPT_IF_NEEDED();
6717	return VINF_SUCCESS;
6718	}
6719
6720	Assert(pVCpu);
6721	Assert(pvUser);
6722	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6723	HMVMX_ASSERT_PREEMPT_SAFE();
6724
6725	VMMRZCallRing3Disable(pVCpu);
6726	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6727
6728	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6729	enmOperation));
6730
6731	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6732	AssertRCReturn(rc, rc);
6733
6734	VMMRZCallRing3Enable(pVCpu);
6735	return VINF_SUCCESS;
6736	}
6737
6738
6739	/**
6740	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6741	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6742	*
6743	* @param pVCpu Pointer to the VMCPU.
6744	*/
6745	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6746	{
6747	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6748	{
6749	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6750	{
6751	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6752	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6753	AssertRC(rc);
6754	}
6755	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6756	}
6757
6758
6759	/**
6760	* Evaluates the event to be delivered to the guest and sets it as the pending
6761	* event.
6762	*
6763	* @param pVCpu Pointer to the VMCPU.
6764	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6765	* out-of-sync. Make sure to update the required fields
6766	* before using them.
6767	*/
6768	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6769	{
6770	Assert(!pVCpu->hm.s.Event.fPending);
6771
6772	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6773	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6774	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6775	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6776
6777	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6778	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6779	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6780	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6781	Assert(!TRPMHasTrap(pVCpu));
6782
6783	/** @todo SMI. SMIs take priority over NMIs. */
6784	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6785	{
6786	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6787	if ( !fBlockMovSS
6788	&& !fBlockSti)
6789	{
6790	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6791	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6792	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6793	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6794
6795	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6796	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6797	}
6798	else
6799	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6800	}
6801	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6802	&& !pVCpu->hm.s.fSingleInstruction)
6803	{
6804	/*
6805	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6806	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6807	* evaluated here and not set as pending, solely based on the force-flags.
6808	*/
6809	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6810	AssertRC(rc);
6811	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6812	if ( !fBlockInt
6813	&& !fBlockSti
6814	&& !fBlockMovSS)
6815	{
6816	uint8_t u8Interrupt;
6817	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6818	if (RT_SUCCESS(rc))
6819	{
6820	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6821	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6822	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6823
6824	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6825	}
6826	else
6827	{
6828	/** @todo Does this actually happen? If not turn it into an assertion. */
6829	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6830	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6831	}
6832	}
6833	else
6834	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6835	}
6836	}
6837
6838
6839	/**
6840	* Injects any pending events into the guest if the guest is in a state to
6841	* receive them.
6842	*
6843	* @returns VBox status code (informational status codes included).
6844	* @param pVCpu Pointer to the VMCPU.
6845	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6846	* out-of-sync. Make sure to update the required fields
6847	* before using them.
6848	*/
6849	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6850	{
6851	HMVMX_ASSERT_PREEMPT_SAFE();
6852	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6853
6854	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6855	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6856	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6857	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6858
6859	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6860	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6861	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6862	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6863	Assert(!TRPMHasTrap(pVCpu));
6864
6865	int rc = VINF_SUCCESS;
6866	if (pVCpu->hm.s.Event.fPending)
6867	{
6868	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
6869	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6870	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6871	{
6872	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6873	AssertRCReturn(rc, rc);
6874	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6875	Assert(!fBlockInt);
6876	Assert(!fBlockSti);
6877	Assert(!fBlockMovSS);
6878	}
6879	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6880	{
6881	Assert(!fBlockSti);
6882	Assert(!fBlockMovSS);
6883	}
6884	#endif
6885	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6886	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6887	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6888	AssertRCReturn(rc, rc);
6889
6890	/* Update the interruptibility-state as it could have been changed by
6891	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6892	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6893	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6894
6895	#ifdef VBOX_WITH_STATISTICS
6896	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6897	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6898	else
6899	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6900	#endif
6901	}
6902
6903	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6904	if ( !pVCpu->hm.s.fSingleInstruction
6905	&& !DBGFIsStepping(pVCpu))
6906	{
6907	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6908	AssertRCReturn(rc2, rc2);
6909	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6910	{
6911	/*
6912	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6913	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6914	* See Intel spec. 27.3.4 "Saving Non-Register State".
6915	*/
6916	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6917	AssertRCReturn(rc2, rc2);
6918	}
6919	}
6920	else
6921	{
6922	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6923	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6924	uIntrState = 0;
6925	}
6926
6927	/*
6928	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6929	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6930	*/
6931	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6932	AssertRC(rc2);
6933
6934	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6935	NOREF(fBlockMovSS); NOREF(fBlockSti);
6936	return rc;
6937	}
6938
6939
6940	/**
6941	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6942	*
6943	* @param pVCpu Pointer to the VMCPU.
6944	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6945	* out-of-sync. Make sure to update the required fields
6946	* before using them.
6947	*/
6948	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6949	{
6950	NOREF(pMixedCtx);
6951	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6952	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6953	}
6954
6955
6956	/**
6957	* Injects a double-fault (#DF) exception into the VM.
6958	*
6959	* @returns VBox status code (informational status code included).
6960	* @param pVCpu Pointer to the VMCPU.
6961	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6962	* out-of-sync. Make sure to update the required fields
6963	* before using them.
6964	*/
6965	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6966	{
6967	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6968	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6969	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6970	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6971	puIntrState);
6972	}
6973
6974
6975	/**
6976	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6977	*
6978	* @param pVCpu Pointer to the VMCPU.
6979	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6980	* out-of-sync. Make sure to update the required fields
6981	* before using them.
6982	*/
6983	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6984	{
6985	NOREF(pMixedCtx);
6986	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6987	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6988	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6989	}
6990
6991
6992	/**
6993	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6994	*
6995	* @param pVCpu Pointer to the VMCPU.
6996	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6997	* out-of-sync. Make sure to update the required fields
6998	* before using them.
6999	* @param cbInstr The value of RIP that is to be pushed on the guest
7000	* stack.
7001	*/
7002	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7003	{
7004	NOREF(pMixedCtx);
7005	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7006	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7007	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7008	}
7009
7010
7011	/**
7012	* Injects a general-protection (#GP) fault into the VM.
7013	*
7014	* @returns VBox status code (informational status code included).
7015	* @param pVCpu Pointer to the VMCPU.
7016	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7017	* out-of-sync. Make sure to update the required fields
7018	* before using them.
7019	* @param u32ErrorCode The error code associated with the #GP.
7020	*/
7021	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7022	uint32_t *puIntrState)
7023	{
7024	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7025	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7026	if (fErrorCodeValid)
7027	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7028	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7029	puIntrState);
7030	}
7031
7032
7033	/**
7034	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7035	*
7036	* @param pVCpu Pointer to the VMCPU.
7037	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7038	* out-of-sync. Make sure to update the required fields
7039	* before using them.
7040	* @param uVector The software interrupt vector number.
7041	* @param cbInstr The value of RIP that is to be pushed on the guest
7042	* stack.
7043	*/
7044	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7045	{
7046	NOREF(pMixedCtx);
7047	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7048	if ( uVector == X86_XCPT_BP
7049	\|\| uVector == X86_XCPT_OF)
7050	{
7051	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7052	}
7053	else
7054	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7055	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7056	}
7057
7058
7059	/**
7060	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7061	* stack.
7062	*
7063	* @returns VBox status code (information status code included).
7064	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7065	* @param pVM Pointer to the VM.
7066	* @param pMixedCtx Pointer to the guest-CPU context.
7067	* @param uValue The value to push to the guest stack.
7068	*/
7069	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7070	{
7071	/*
7072	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7073	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7074	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7075	*/
7076	if (pMixedCtx->sp == 1)
7077	return VINF_EM_RESET;
7078	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7079	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7080	AssertRCReturn(rc, rc);
7081	return rc;
7082	}
7083
7084
7085	/**
7086	* Injects an event into the guest upon VM-entry by updating the relevant fields
7087	* in the VM-entry area in the VMCS.
7088	*
7089	* @returns VBox status code (informational error codes included).
7090	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7091	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7092	*
7093	* @param pVCpu Pointer to the VMCPU.
7094	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7095	* be out-of-sync. Make sure to update the required
7096	* fields before using them.
7097	* @param u64IntInfo The VM-entry interruption-information field.
7098	* @param cbInstr The VM-entry instruction length in bytes (for
7099	* software interrupts, exceptions and privileged
7100	* software exceptions).
7101	* @param u32ErrCode The VM-entry exception error code.
7102	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7103	* @param puIntrState Pointer to the current guest interruptibility-state.
7104	* This interruptibility-state will be updated if
7105	* necessary. This cannot not be NULL.
7106	*
7107	* @remarks Requires CR0!
7108	* @remarks No-long-jump zone!!!
7109	*/
7110	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7111	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7112	{
7113	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7114	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7115	Assert(puIntrState);
7116	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7117
7118	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7119	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7120
7121	#ifdef VBOX_STRICT
7122	/* Validate the error-code-valid bit for hardware exceptions. */
7123	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7124	{
7125	switch (uVector)
7126	{
7127	case X86_XCPT_PF:
7128	case X86_XCPT_DF:
7129	case X86_XCPT_TS:
7130	case X86_XCPT_NP:
7131	case X86_XCPT_SS:
7132	case X86_XCPT_GP:
7133	case X86_XCPT_AC:
7134	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7135	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7136	/* fallthru */
7137	default:
7138	break;
7139	}
7140	}
7141	#endif
7142
7143	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7144	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7145	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7146
7147	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7148
7149	/* We require CR0 to check if the guest is in real-mode. */
7150	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7151	AssertRCReturn(rc, rc);
7152
7153	/*
7154	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7155	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7156	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7157	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7158	*/
7159	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7160	{
7161	PVM pVM = pVCpu->CTX_SUFF(pVM);
7162	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7163	{
7164	Assert(PDMVmmDevHeapIsEnabled(pVM));
7165	Assert(pVM->hm.s.vmx.pRealModeTSS);
7166
7167	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7168	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7169	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7170	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7171	AssertRCReturn(rc, rc);
7172	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7173
7174	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7175	const size_t cbIdtEntry = sizeof(X86IDTR16);
7176	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7177	{
7178	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7179	if (uVector == X86_XCPT_DF)
7180	return VINF_EM_RESET;
7181	else if (uVector == X86_XCPT_GP)
7182	{
7183	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7184	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7185	}
7186
7187	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7188	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7189	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7190	}
7191
7192	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7193	uint16_t uGuestIp = pMixedCtx->ip;
7194	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7195	{
7196	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7197	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7198	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7199	}
7200	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7201	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7202
7203	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7204	X86IDTR16 IdtEntry;
7205	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7206	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7207	AssertRCReturn(rc, rc);
7208
7209	/* Construct the stack frame for the interrupt/exception handler. */
7210	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7211	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7212	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7213	AssertRCReturn(rc, rc);
7214
7215	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7216	if (rc == VINF_SUCCESS)
7217	{
7218	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7219	pMixedCtx->rip = IdtEntry.offSel;
7220	pMixedCtx->cs.Sel = IdtEntry.uSel;
7221	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7222	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7223	&& uVector == X86_XCPT_PF)
7224	{
7225	pMixedCtx->cr2 = GCPtrFaultAddress;
7226	}
7227
7228	/* If any other guest-state bits are changed here, make sure to update
7229	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7230	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7231	\| HM_CHANGED_GUEST_RIP
7232	\| HM_CHANGED_GUEST_RFLAGS
7233	\| HM_CHANGED_GUEST_RSP);
7234
7235	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7236	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7237	{
7238	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7239	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7240	Log4(("Clearing inhibition due to STI.\n"));
7241	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7242	}
7243	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7244
7245	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7246	it, if we are returning to ring-3 before executing guest code. */
7247	pVCpu->hm.s.Event.fPending = false;
7248	}
7249	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7250	return rc;
7251	}
7252	else
7253	{
7254	/*
7255	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7256	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7257	*/
7258	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7259	}
7260	}
7261
7262	/* Validate. */
7263	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7264	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7265	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7266
7267	/* Inject. */
7268	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7269	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7270	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7271	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7272
7273	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7274	&& uVector == X86_XCPT_PF)
7275	{
7276	pMixedCtx->cr2 = GCPtrFaultAddress;
7277	}
7278
7279	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7280	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7281
7282	AssertRCReturn(rc, rc);
7283	return rc;
7284	}
7285
7286
7287	/**
7288	* Clears the interrupt-window exiting control in the VMCS and if necessary
7289	* clears the current event in the VMCS as well.
7290	*
7291	* @returns VBox status code.
7292	* @param pVCpu Pointer to the VMCPU.
7293	*
7294	* @remarks Use this function only to clear events that have not yet been
7295	* delivered to the guest but are injected in the VMCS!
7296	* @remarks No-long-jump zone!!!
7297	*/
7298	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7299	{
7300	int rc;
7301	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7302
7303	/* Clear interrupt-window exiting control. */
7304	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7305	{
7306	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7307	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7308	AssertRC(rc);
7309	}
7310
7311	if (!pVCpu->hm.s.Event.fPending)
7312	return;
7313
7314	#ifdef VBOX_STRICT
7315	uint32_t u32EntryInfo;
7316	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7317	AssertRC(rc);
7318	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7319	#endif
7320
7321	/* Clear the entry-interruption field (including the valid bit). */
7322	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7323	AssertRC(rc);
7324
7325	/* Clear the pending debug exception field. */
7326	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7327	AssertRC(rc);
7328	}
7329
7330
7331	/**
7332	* Enters the VT-x session.
7333	*
7334	* @returns VBox status code.
7335	* @param pVM Pointer to the VM.
7336	* @param pVCpu Pointer to the VMCPU.
7337	* @param pCpu Pointer to the CPU info struct.
7338	*/
7339	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7340	{
7341	AssertPtr(pVM);
7342	AssertPtr(pVCpu);
7343	Assert(pVM->hm.s.vmx.fSupported);
7344	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7345	NOREF(pCpu); NOREF(pVM);
7346
7347	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7348	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7349
7350	#ifdef VBOX_STRICT
7351	/* Make sure we're in VMX root mode. */
7352	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7353	if (!(u32HostCR4 & X86_CR4_VMXE))
7354	{
7355	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7356	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7357	}
7358	#endif
7359
7360	/*
7361	* Load the VCPU's VMCS as the current (and active) one.
7362	*/
7363	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7364	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7365	if (RT_FAILURE(rc))
7366	return rc;
7367
7368	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7369	pVCpu->hm.s.fLeaveDone = false;
7370	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7371
7372	return VINF_SUCCESS;
7373	}
7374
7375
7376	/**
7377	* The thread-context callback (only on platforms which support it).
7378	*
7379	* @param enmEvent The thread-context event.
7380	* @param pVCpu Pointer to the VMCPU.
7381	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7382	* @thread EMT(pVCpu)
7383	*/
7384	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7385	{
7386	NOREF(fGlobalInit);
7387
7388	switch (enmEvent)
7389	{
7390	case RTTHREADCTXEVENT_PREEMPTING:
7391	{
7392	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7393	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7394	VMCPU_ASSERT_EMT(pVCpu);
7395
7396	PVM pVM = pVCpu->CTX_SUFF(pVM);
7397	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7398
7399	/* No longjmps (logger flushes, locks) in this fragile context. */
7400	VMMRZCallRing3Disable(pVCpu);
7401	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7402
7403	/*
7404	* Restore host-state (FPU, debug etc.)
7405	*/
7406	if (!pVCpu->hm.s.fLeaveDone)
7407	{
7408	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7409	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7410	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7411	pVCpu->hm.s.fLeaveDone = true;
7412	}
7413
7414	/* Leave HM context, takes care of local init (term). */
7415	int rc = HMR0LeaveCpu(pVCpu);
7416	AssertRC(rc); NOREF(rc);
7417
7418	/* Restore longjmp state. */
7419	VMMRZCallRing3Enable(pVCpu);
7420	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7421	break;
7422	}
7423
7424	case RTTHREADCTXEVENT_RESUMED:
7425	{
7426	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7427	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7428	VMCPU_ASSERT_EMT(pVCpu);
7429
7430	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7431	VMMRZCallRing3Disable(pVCpu);
7432	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7433
7434	/* Initialize the bare minimum state required for HM. This takes care of
7435	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7436	int rc = HMR0EnterCpu(pVCpu);
7437	AssertRC(rc);
7438	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7439
7440	/* Load the active VMCS as the current one. */
7441	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7442	{
7443	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7444	AssertRC(rc); NOREF(rc);
7445	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7446	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7447	}
7448	pVCpu->hm.s.fLeaveDone = false;
7449
7450	/* Restore longjmp state. */
7451	VMMRZCallRing3Enable(pVCpu);
7452	break;
7453	}
7454
7455	default:
7456	break;
7457	}
7458	}
7459
7460
7461	/**
7462	* Saves the host state in the VMCS host-state.
7463	* Sets up the VM-exit MSR-load area.
7464	*
7465	* The CPU state will be loaded from these fields on every successful VM-exit.
7466	*
7467	* @returns VBox status code.
7468	* @param pVM Pointer to the VM.
7469	* @param pVCpu Pointer to the VMCPU.
7470	*
7471	* @remarks No-long-jump zone!!!
7472	*/
7473	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7474	{
7475	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7476
7477	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7478	return VINF_SUCCESS;
7479
7480	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7481	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7482
7483	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7484	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7485
7486	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7487	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7488
7489	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7490	return rc;
7491	}
7492
7493
7494	/**
7495	* Saves the host state in the VMCS host-state.
7496	*
7497	* @returns VBox status code.
7498	* @param pVM Pointer to the VM.
7499	* @param pVCpu Pointer to the VMCPU.
7500	*
7501	* @remarks No-long-jump zone!!!
7502	*/
7503	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7504	{
7505	AssertPtr(pVM);
7506	AssertPtr(pVCpu);
7507
7508	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7509
7510	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7511	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7513	return hmR0VmxSaveHostState(pVM, pVCpu);
7514	}
7515
7516
7517	/**
7518	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7519	* loaded from these fields on every successful VM-entry.
7520	*
7521	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7522	* Sets up the VM-entry controls.
7523	* Sets up the appropriate VMX non-root function to execute guest code based on
7524	* the guest CPU mode.
7525	*
7526	* @returns VBox status code.
7527	* @param pVM Pointer to the VM.
7528	* @param pVCpu Pointer to the VMCPU.
7529	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7530	* out-of-sync. Make sure to update the required fields
7531	* before using them.
7532	*
7533	* @remarks No-long-jump zone!!!
7534	*/
7535	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7536	{
7537	AssertPtr(pVM);
7538	AssertPtr(pVCpu);
7539	AssertPtr(pMixedCtx);
7540	HMVMX_ASSERT_PREEMPT_SAFE();
7541
7542	#ifdef LOG_ENABLED
7543	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7544	* probably not initialized yet? Anyway this will do for now.
7545	*
7546	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7547	* interface and disable ring-3 calls when thread-context hooks are not
7548	* available. */
7549	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7550	VMMR0LogFlushDisable(pVCpu);
7551	#endif
7552
7553	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7554
7555	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7556
7557	/* Determine real-on-v86 mode. */
7558	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7559	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7560	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7561	{
7562	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7563	}
7564
7565	/*
7566	* Load the guest-state into the VMCS.
7567	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7568	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7569	*/
7570	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7571	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7572
7573	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7574	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7575	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7576
7577	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7578	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7579	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7580
7581	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7582	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7583
7584	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7585	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7586
7587	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7588	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7589	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7590
7591	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7592	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7593
7594	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7595	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7596
7597	/*
7598	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7599	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7600	*/
7601	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7602	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7603
7604	/* Clear any unused and reserved bits. */
7605	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7606
7607	#ifdef LOG_ENABLED
7608	/* Only reenable log-flushing if the caller has it enabled. */
7609	if (!fCallerDisabledLogFlush)
7610	VMMR0LogFlushEnable(pVCpu);
7611	#endif
7612
7613	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7614	return rc;
7615	}
7616
7617
7618	/**
7619	* Loads the state shared between the host and guest into the VMCS.
7620	*
7621	* @param pVM Pointer to the VM.
7622	* @param pVCpu Pointer to the VMCPU.
7623	* @param pCtx Pointer to the guest-CPU context.
7624	*
7625	* @remarks No-long-jump zone!!!
7626	*/
7627	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7628	{
7629	NOREF(pVM);
7630
7631	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7632	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7633
7634	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7635	{
7636	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7637	AssertRC(rc);
7638	}
7639
7640	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7641	{
7642	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7643	AssertRC(rc);
7644
7645	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7646	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7647	{
7648	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7649	AssertRC(rc);
7650	}
7651	}
7652
7653	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7654	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7655	}
7656
7657
7658	/**
7659	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7660	*
7661	* @param pVM Pointer to the VM.
7662	* @param pVCpu Pointer to the VMCPU.
7663	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7664	* out-of-sync. Make sure to update the required fields
7665	* before using them.
7666	*/
7667	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7668	{
7669	HMVMX_ASSERT_PREEMPT_SAFE();
7670
7671	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7672	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7673	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7674	#endif
7675
7676	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7677	{
7678	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7679	AssertRC(rc);
7680	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7681	}
7682	else if (VMCPU_HMCF_VALUE(pVCpu))
7683	{
7684	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7685	AssertRC(rc);
7686	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7687	}
7688
7689	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7690	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7691	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7692	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7693
7694	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7695	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7696	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7697	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7698	#endif
7699	}
7700
7701
7702	/**
7703	* Does the preparations before executing guest code in VT-x.
7704	*
7705	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7706	* recompiler. We must be cautious what we do here regarding committing
7707	* guest-state information into the VMCS assuming we assuredly execute the
7708	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7709	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7710	* so that the recompiler can (and should) use them when it resumes guest
7711	* execution. Otherwise such operations must be done when we can no longer
7712	* exit to ring-3.
7713	*
7714	* @returns Strict VBox status code.
7715	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7716	* have been disabled.
7717	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7718	* double-fault into the guest.
7719	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7720	*
7721	* @param pVM Pointer to the VM.
7722	* @param pVCpu Pointer to the VMCPU.
7723	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7724	* out-of-sync. Make sure to update the required fields
7725	* before using them.
7726	* @param pVmxTransient Pointer to the VMX transient structure.
7727	*
7728	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7729	* interrupts will be disabled.
7730	*/
7731	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7732	{
7733	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7734
7735	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7736	PGMRZDynMapFlushAutoSet(pVCpu);
7737	#endif
7738
7739	/* Check force flag actions that might require us to go back to ring-3. */
7740	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7741	if (rc != VINF_SUCCESS)
7742	return rc;
7743
7744	#ifndef IEM_VERIFICATION_MODE_FULL
7745	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7746	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7747	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7748	{
7749	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7750	RTGCPHYS GCPhysApicBase;
7751	GCPhysApicBase = pMixedCtx->msrApicBase;
7752	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7753
7754	/* Unalias any existing mapping. */
7755	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7756	AssertRCReturn(rc, rc);
7757
7758	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7759	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7760	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7761	AssertRCReturn(rc, rc);
7762
7763	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7764	}
7765	#endif /* !IEM_VERIFICATION_MODE_FULL */
7766
7767	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7768	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7769
7770	/*
7771	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7772	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7773	*/
7774	if (TRPMHasTrap(pVCpu))
7775	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7776	else if (!pVCpu->hm.s.Event.fPending)
7777	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7778
7779	/*
7780	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7781	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7782	*/
7783	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7784	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7785	{
7786	Assert(rc == VINF_EM_RESET);
7787	return rc;
7788	}
7789
7790	/*
7791	* No longjmps to ring-3 from this point on!!!
7792	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7793	* This also disables flushing of the R0-logger instance (if any).
7794	*/
7795	VMMRZCallRing3Disable(pVCpu);
7796
7797	/*
7798	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7799	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7800	*
7801	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7802	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7803	*
7804	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7805	* executing guest code.
7806	*/
7807	pVmxTransient->uEflags = ASMIntDisableFlags();
7808	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7809	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7810	{
7811	hmR0VmxClearEventVmcs(pVCpu);
7812	ASMSetFlags(pVmxTransient->uEflags);
7813	VMMRZCallRing3Enable(pVCpu);
7814	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7815	return VINF_EM_RAW_TO_R3;
7816	}
7817	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7818	{
7819	hmR0VmxClearEventVmcs(pVCpu);
7820	ASMSetFlags(pVmxTransient->uEflags);
7821	VMMRZCallRing3Enable(pVCpu);
7822	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7823	return VINF_EM_RAW_INTERRUPT;
7824	}
7825
7826	/* We've injected any pending events. This is really the point of no return (to ring-3). */
7827	pVCpu->hm.s.Event.fPending = false;
7828
7829	return VINF_SUCCESS;
7830	}
7831
7832
7833	/**
7834	* Prepares to run guest code in VT-x and we've committed to doing so. This
7835	* means there is no backing out to ring-3 or anywhere else at this
7836	* point.
7837	*
7838	* @param pVM Pointer to the VM.
7839	* @param pVCpu Pointer to the VMCPU.
7840	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7841	* out-of-sync. Make sure to update the required fields
7842	* before using them.
7843	* @param pVmxTransient Pointer to the VMX transient structure.
7844	*
7845	* @remarks Called with preemption disabled.
7846	* @remarks No-long-jump zone!!!
7847	*/
7848	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7849	{
7850	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7851	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7852	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7853
7854	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7855	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7856
7857	/*
7858	* If we are injecting events to a real-on-v86 mode guest, we may have to update
7859	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7860	* Reload only the necessary state, the assertion will catch if other parts of the code
7861	* change.
7862	*/
7863	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7864	{
7865	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7866	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7867	}
7868
7869	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7870	if (!CPUMIsGuestFPUStateActive(pVCpu))
7871	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7872	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7873	#endif
7874
7875	if ( pVCpu->hm.s.fUseGuestFpu
7876	&& !CPUMIsGuestFPUStateActive(pVCpu))
7877	{
7878	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7879	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
7880	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7881	}
7882
7883	/*
7884	* The host MSR values the very first time around won't be updated, so we need to
7885	* fill those values in. Subsequently, it's updated as part of the host state.
7886	*/
7887	if (!pVCpu->hm.s.vmx.fUpdatedHostMsrs)
7888	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
7889
7890	/*
7891	* Load the host state bits as we may've been preempted (only happens when
7892	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7893	*/
7894	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7895	{
7896	/* This ASSUMES that pfnStartVM has been set up already. */
7897	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7898	AssertRC(rc);
7899	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7900	}
7901	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7902
7903	/*
7904	* Load the state shared between host and guest (FPU, debug).
7905	*/
7906	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7907	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7908	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7909
7910	/* Store status of the shared guest-host state at the time of VM-entry. */
7911	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7912	if (CPUMIsGuestInLongModeEx(pMixedCtx))
7913	{
7914	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7915	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7916	}
7917	else
7918	#endif
7919	{
7920	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7921	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7922	}
7923	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7924
7925	/*
7926	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7927	*/
7928	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7929	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7930
7931	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7932	RTCPUID idCurrentCpu = pCpu->idCpu;
7933	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7934	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
7935	{
7936	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
7937	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7938	}
7939
7940	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7941	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7942	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7943	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7944
7945	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7946
7947	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7948	to start executing. */
7949
7950	/*
7951	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
7952	*/
7953	uint64_t uGuestTscAuxMsr;
7954	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7955	{
7956	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7957	{
7958	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &uGuestTscAuxMsr);
7959	AssertRC(rc2);
7960	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, uGuestTscAuxMsr, true /* fUpdateHostMsr */);
7961	}
7962	else
7963	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
7964	}
7965	#ifdef VBOX_STRICT
7966	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
7967	#endif
7968	}
7969
7970
7971	/**
7972	* Performs some essential restoration of state after running guest code in
7973	* VT-x.
7974	*
7975	* @param pVM Pointer to the VM.
7976	* @param pVCpu Pointer to the VMCPU.
7977	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7978	* out-of-sync. Make sure to update the required fields
7979	* before using them.
7980	* @param pVmxTransient Pointer to the VMX transient structure.
7981	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7982	*
7983	* @remarks Called with interrupts disabled, and returns with interrups enabled!
7984	*
7985	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7986	* unconditionally when it is safe to do so.
7987	*/
7988	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7989	{
7990	NOREF(pVM);
7991
7992	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7993
7994	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7995	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7996	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7997	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7998	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7999
8000	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8001	{
8002	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8003	{
8004	/* VT-x restored the host TSC_AUX MSR for us, update the guest value from the VMCS area
8005	if it could have changed without causing a VM-exit. */
8006	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
8007	{
8008	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8009	AssertRC(rc2);
8010	}
8011	}
8012
8013	/** @todo Find a way to fix hardcoding a guestimate. */
8014	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8015	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8016	}
8017
8018	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8019	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8020	Assert(!(ASMGetFlags() & X86_EFL_IF));
8021	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8022
8023	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8024	if (CPUMIsGuestFPUStateActive(pVCpu))
8025	{
8026	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8027	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8028	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8029	}
8030	#endif
8031
8032	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8033	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8034	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8035	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8036
8037	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8038	uint32_t uExitReason;
8039	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8040	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8041	AssertRC(rc);
8042	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8043	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8044
8045	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8046	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8047	{
8048	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8049	pVmxTransient->fVMEntryFailed));
8050	return;
8051	}
8052
8053	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8054	{
8055	/* Update the guest interruptibility-state from the VMCS. */
8056	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8057	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8058	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8059	AssertRC(rc);
8060	#endif
8061	/*
8062	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8063	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8064	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8065	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8066	*/
8067	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8068	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8069	{
8070	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8071	AssertRC(rc);
8072	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8073	}
8074	}
8075	}
8076
8077
8078
8079	/**
8080	* Runs the guest code using VT-x the normal way.
8081	*
8082	* @returns VBox status code.
8083	* @param pVM Pointer to the VM.
8084	* @param pVCpu Pointer to the VMCPU.
8085	* @param pCtx Pointer to the guest-CPU context.
8086	*
8087	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
8088	* @remarks Called with preemption disabled.
8089	*/
8090	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8091	{
8092	VMXTRANSIENT VmxTransient;
8093	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8094	int rc = VERR_INTERNAL_ERROR_5;
8095	uint32_t cLoops = 0;
8096
8097	for (;; cLoops++)
8098	{
8099	Assert(!HMR0SuspendPending());
8100	HMVMX_ASSERT_CPU_SAFE();
8101
8102	/* Preparatory work for running guest code, this may force us to return
8103	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8104	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8105	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8106	if (rc != VINF_SUCCESS)
8107	break;
8108
8109	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8110	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8111	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8112
8113	/* Restore any residual host-state and save any bits shared between host
8114	and guest into the guest-CPU state. Re-enables interrupts! */
8115	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8116
8117	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8118	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8119	{
8120	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8121	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8122	return rc;
8123	}
8124
8125	/* Handle the VM-exit. */
8126	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8127	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8128	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8129	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8130	HMVMX_START_EXIT_DISPATCH_PROF();
8131	#ifdef HMVMX_USE_FUNCTION_TABLE
8132	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8133	#else
8134	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8135	#endif
8136	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8137	if (rc != VINF_SUCCESS)
8138	break;
8139	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8140	{
8141	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8142	rc = VINF_EM_RAW_INTERRUPT;
8143	break;
8144	}
8145	}
8146
8147	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8148	return rc;
8149	}
8150
8151
8152	/**
8153	* Single steps guest code using VT-x.
8154	*
8155	* @returns VBox status code.
8156	* @param pVM Pointer to the VM.
8157	* @param pVCpu Pointer to the VMCPU.
8158	* @param pCtx Pointer to the guest-CPU context.
8159	*
8160	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
8161	* @remarks Called with preemption disabled.
8162	*/
8163	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8164	{
8165	VMXTRANSIENT VmxTransient;
8166	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8167	int rc = VERR_INTERNAL_ERROR_5;
8168	uint32_t cLoops = 0;
8169	uint16_t uCsStart = pCtx->cs.Sel;
8170	uint64_t uRipStart = pCtx->rip;
8171
8172	for (;; cLoops++)
8173	{
8174	Assert(!HMR0SuspendPending());
8175	HMVMX_ASSERT_CPU_SAFE();
8176
8177	/* Preparatory work for running guest code, this may force us to return
8178	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8179	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8180	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8181	if (rc != VINF_SUCCESS)
8182	break;
8183
8184	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8185	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8186	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8187
8188	/* Restore any residual host-state and save any bits shared between host
8189	and guest into the guest-CPU state. Re-enables interrupts! */
8190	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8191
8192	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8193	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8194	{
8195	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8196	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8197	return rc;
8198	}
8199
8200	/* Handle the VM-exit. */
8201	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8202	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8203	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8204	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8205	HMVMX_START_EXIT_DISPATCH_PROF();
8206	#ifdef HMVMX_USE_FUNCTION_TABLE
8207	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8208	#else
8209	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8210	#endif
8211	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8212	if (rc != VINF_SUCCESS)
8213	break;
8214	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8215	{
8216	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8217	rc = VINF_EM_RAW_INTERRUPT;
8218	break;
8219	}
8220
8221	/*
8222	* Did the RIP change, if so, consider it a single step.
8223	* Otherwise, make sure one of the TFs gets set.
8224	*/
8225	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8226	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8227	AssertRCReturn(rc2, rc2);
8228	if ( pCtx->rip != uRipStart
8229	\|\| pCtx->cs.Sel != uCsStart)
8230	{
8231	rc = VINF_EM_DBG_STEPPED;
8232	break;
8233	}
8234	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8235	}
8236
8237	/*
8238	* Clear the X86_EFL_TF if necessary.
8239	*/
8240	if (pVCpu->hm.s.fClearTrapFlag)
8241	{
8242	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8243	AssertRCReturn(rc2, rc2);
8244	pVCpu->hm.s.fClearTrapFlag = false;
8245	pCtx->eflags.Bits.u1TF = 0;
8246	}
8247	/** @todo there seems to be issues with the resume flag when the monitor trap
8248	* flag is pending without being used. Seen early in bios init when
8249	* accessing APIC page in prot mode. */
8250
8251	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8252	return rc;
8253	}
8254
8255
8256	/**
8257	* Runs the guest code using VT-x.
8258	*
8259	* @returns VBox status code.
8260	* @param pVM Pointer to the VM.
8261	* @param pVCpu Pointer to the VMCPU.
8262	* @param pCtx Pointer to the guest-CPU context.
8263	*
8264	* @remarks Called with preemption disabled.
8265	*/
8266	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8267	{
8268	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8269	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8270	HMVMX_ASSERT_PREEMPT_SAFE();
8271
8272	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8273
8274	int rc;
8275	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8276	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8277	else
8278	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8279
8280	if (rc == VERR_EM_INTERPRETER)
8281	rc = VINF_EM_RAW_EMULATE_INSTR;
8282	else if (rc == VINF_EM_RESET)
8283	rc = VINF_EM_TRIPLE_FAULT;
8284
8285	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8286	if (RT_FAILURE(rc2))
8287	{
8288	pVCpu->hm.s.u32HMError = rc;
8289	rc = rc2;
8290	}
8291	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8292	return rc;
8293	}
8294
8295
8296	#ifndef HMVMX_USE_FUNCTION_TABLE
8297	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8298	{
8299	#ifdef DEBUG_ramshankar
8300	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8301	# define LDVMCS() do { VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8302	#endif
8303	int rc;
8304	switch (rcReason)
8305	{
8306	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8307	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8308	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8309	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8310	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8311	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8312	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8313	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8314	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8315	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8316	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8317	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8318	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8319	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8320	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8321	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8322	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8323	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8324	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8325	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8326	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8327	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8328	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8329	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8330	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8331	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8332	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8333	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8334	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8335	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8336	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8337	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8338	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8339
8340	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8341	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8342	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8343	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8344	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8345	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8346	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8347	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8348	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8349
8350	case VMX_EXIT_VMCALL:
8351	case VMX_EXIT_VMCLEAR:
8352	case VMX_EXIT_VMLAUNCH:
8353	case VMX_EXIT_VMPTRLD:
8354	case VMX_EXIT_VMPTRST:
8355	case VMX_EXIT_VMREAD:
8356	case VMX_EXIT_VMRESUME:
8357	case VMX_EXIT_VMWRITE:
8358	case VMX_EXIT_VMXOFF:
8359	case VMX_EXIT_VMXON:
8360	case VMX_EXIT_INVEPT:
8361	case VMX_EXIT_INVVPID:
8362	case VMX_EXIT_VMFUNC:
8363	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8364	break;
8365	default:
8366	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8367	break;
8368	}
8369	return rc;
8370	}
8371	#endif
8372
8373	#ifdef DEBUG
8374	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8375	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8376	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8377
8378	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8379	do \
8380	{ \
8381	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8382	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8383	} while (0)
8384
8385	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8386	do { \
8387	AssertPtr(pVCpu); \
8388	AssertPtr(pMixedCtx); \
8389	AssertPtr(pVmxTransient); \
8390	Assert(pVmxTransient->fVMEntryFailed == false); \
8391	Assert(ASMIntAreEnabled()); \
8392	HMVMX_ASSERT_PREEMPT_SAFE(); \
8393	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8394	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8395	HMVMX_ASSERT_PREEMPT_SAFE(); \
8396	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8397	HMVMX_ASSERT_PREEMPT_CPUID(); \
8398	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8399	} while (0)
8400
8401	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8402	do { \
8403	Log4Func(("\n")); \
8404	} while (0)
8405	#else /* Release builds */
8406	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8407	do { \
8408	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8409	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8410	} while (0)
8411	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8412	#endif
8413
8414
8415	/**
8416	* Advances the guest RIP after reading it from the VMCS.
8417	*
8418	* @returns VBox status code.
8419	* @param pVCpu Pointer to the VMCPU.
8420	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8421	* out-of-sync. Make sure to update the required fields
8422	* before using them.
8423	* @param pVmxTransient Pointer to the VMX transient structure.
8424	*
8425	* @remarks No-long-jump zone!!!
8426	*/
8427	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8428	{
8429	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8430	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8431	AssertRCReturn(rc, rc);
8432
8433	pMixedCtx->rip += pVmxTransient->cbInstr;
8434	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8435	return rc;
8436	}
8437
8438
8439	/**
8440	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8441	* and update error record fields accordingly.
8442	*
8443	* @return VMX_IGS_* return codes.
8444	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8445	* wrong with the guest state.
8446	*
8447	* @param pVM Pointer to the VM.
8448	* @param pVCpu Pointer to the VMCPU.
8449	* @param pCtx Pointer to the guest-CPU state.
8450	*/
8451	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8452	{
8453	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8454	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8455	uError = (err); \
8456	break; \
8457	} else do { } while (0)
8458	/* Duplicate of IEM_IS_CANONICAL(). */
8459	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8460
8461	int rc;
8462	uint32_t uError = VMX_IGS_ERROR;
8463	uint32_t u32Val;
8464	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8465
8466	do
8467	{
8468	/*
8469	* CR0.
8470	*/
8471	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8472	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8473	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8474	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8475	if (fUnrestrictedGuest)
8476	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8477
8478	uint32_t u32GuestCR0;
8479	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8480	AssertRCBreak(rc);
8481	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8482	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8483	if ( !fUnrestrictedGuest
8484	&& (u32GuestCR0 & X86_CR0_PG)
8485	&& !(u32GuestCR0 & X86_CR0_PE))
8486	{
8487	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8488	}
8489
8490	/*
8491	* CR4.
8492	*/
8493	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8494	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8495
8496	uint32_t u32GuestCR4;
8497	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8498	AssertRCBreak(rc);
8499	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8500	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8501
8502	/*
8503	* IA32_DEBUGCTL MSR.
8504	*/
8505	uint64_t u64Val;
8506	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8507	AssertRCBreak(rc);
8508	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8509	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8510	{
8511	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8512	}
8513	uint64_t u64DebugCtlMsr = u64Val;
8514
8515	#ifdef VBOX_STRICT
8516	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8517	AssertRCBreak(rc);
8518	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8519	#endif
8520	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8521
8522	/*
8523	* RIP and RFLAGS.
8524	*/
8525	uint32_t u32Eflags;
8526	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8527	if (HMVMX_IS_64BIT_HOST_MODE())
8528	{
8529	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8530	AssertRCBreak(rc);
8531	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8532	if ( !fLongModeGuest
8533	\|\| !pCtx->cs.Attr.n.u1Long)
8534	{
8535	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8536	}
8537	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8538	* must be identical if the "IA32e mode guest" VM-entry control is 1
8539	* and CS.L is 1. No check applies if the CPU supports 64
8540	* linear-address bits. */
8541
8542	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8543	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8544	AssertRCBreak(rc);
8545	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8546	VMX_IGS_RFLAGS_RESERVED);
8547	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8548	u32Eflags = u64Val;
8549	}
8550	else
8551	#endif
8552	{
8553	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8554	AssertRCBreak(rc);
8555	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8556	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8557	}
8558
8559	if ( fLongModeGuest
8560	\|\| ( fUnrestrictedGuest
8561	&& !(u32GuestCR0 & X86_CR0_PE)))
8562	{
8563	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8564	}
8565
8566	uint32_t u32EntryInfo;
8567	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8568	AssertRCBreak(rc);
8569	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8570	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8571	{
8572	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8573	}
8574
8575	/*
8576	* 64-bit checks.
8577	*/
8578	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8579	if (HMVMX_IS_64BIT_HOST_MODE())
8580	{
8581	if ( fLongModeGuest
8582	&& !fUnrestrictedGuest)
8583	{
8584	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8585	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8586	}
8587
8588	if ( !fLongModeGuest
8589	&& (u32GuestCR4 & X86_CR4_PCIDE))
8590	{
8591	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8592	}
8593
8594	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8595	* 51:32 beyond the processor's physical-address width are 0. */
8596
8597	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8598	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8599	{
8600	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8601	}
8602
8603	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8604	AssertRCBreak(rc);
8605	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8606
8607	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8608	AssertRCBreak(rc);
8609	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8610	}
8611	#endif
8612
8613	/*
8614	* PERF_GLOBAL MSR.
8615	*/
8616	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8617	{
8618	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8619	AssertRCBreak(rc);
8620	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8621	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8622	}
8623
8624	/*
8625	* PAT MSR.
8626	*/
8627	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8628	{
8629	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8630	AssertRCBreak(rc);
8631	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8632	for (unsigned i = 0; i < 8; i++)
8633	{
8634	uint8_t u8Val = (u64Val & 0x7);
8635	if ( u8Val != 0 /* UC */
8636	\|\| u8Val != 1 /* WC */
8637	\|\| u8Val != 4 /* WT */
8638	\|\| u8Val != 5 /* WP */
8639	\|\| u8Val != 6 /* WB */
8640	\|\| u8Val != 7 /* UC- */)
8641	{
8642	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8643	}
8644	u64Val >>= 3;
8645	}
8646	}
8647
8648	/*
8649	* EFER MSR.
8650	*/
8651	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8652	{
8653	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8654	AssertRCBreak(rc);
8655	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8656	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8657	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8658	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8659	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8660	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8661	}
8662
8663	/*
8664	* Segment registers.
8665	*/
8666	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8667	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8668	if (!(u32Eflags & X86_EFL_VM))
8669	{
8670	/* CS */
8671	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8672	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8673	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8674	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8675	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8676	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8677	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8678	/* CS cannot be loaded with NULL in protected mode. */
8679	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8680	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8681	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8682	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8683	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8684	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8685	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8686	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8687	else
8688	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8689
8690	/* SS */
8691	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8692	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8693	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8694	if ( !(pCtx->cr0 & X86_CR0_PE)
8695	\|\| pCtx->cs.Attr.n.u4Type == 3)
8696	{
8697	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8698	}
8699	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8700	{
8701	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8702	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8703	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8704	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8705	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8706	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8707	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8708	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8709	}
8710
8711	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8712	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8713	{
8714	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8715	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8716	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8717	\|\| pCtx->ds.Attr.n.u4Type > 11
8718	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8719	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8720	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8721	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8722	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8723	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8724	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8725	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8726	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8727	}
8728	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8729	{
8730	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8731	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8732	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8733	\|\| pCtx->es.Attr.n.u4Type > 11
8734	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8735	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8736	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8737	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8738	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8739	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8740	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8741	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8742	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8743	}
8744	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8745	{
8746	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8747	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8748	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8749	\|\| pCtx->fs.Attr.n.u4Type > 11
8750	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8751	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8752	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8753	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8754	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8755	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8756	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8757	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8758	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8759	}
8760	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8761	{
8762	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8763	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8764	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8765	\|\| pCtx->gs.Attr.n.u4Type > 11
8766	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8767	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8768	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8769	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8770	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8771	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8772	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8773	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8774	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8775	}
8776	/* 64-bit capable CPUs. */
8777	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8778	if (HMVMX_IS_64BIT_HOST_MODE())
8779	{
8780	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8781	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8782	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8783	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8784	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8785	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8786	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8787	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8788	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8789	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8790	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8791	}
8792	#endif
8793	}
8794	else
8795	{
8796	/* V86 mode checks. */
8797	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8798	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8799	{
8800	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8801	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8802	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8803	}
8804	else
8805	{
8806	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8807	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8808	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8809	}
8810
8811	/* CS */
8812	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8813	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8814	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8815	/* SS */
8816	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8817	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8818	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8819	/* DS */
8820	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8821	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8822	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8823	/* ES */
8824	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8825	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8826	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8827	/* FS */
8828	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8829	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8830	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8831	/* GS */
8832	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8833	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8834	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8835	/* 64-bit capable CPUs. */
8836	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8837	if (HMVMX_IS_64BIT_HOST_MODE())
8838	{
8839	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8840	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8841	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8842	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8843	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8844	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8845	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8846	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8847	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8848	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8849	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8850	}
8851	#endif
8852	}
8853
8854	/*
8855	* TR.
8856	*/
8857	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8858	/* 64-bit capable CPUs. */
8859	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8860	if (HMVMX_IS_64BIT_HOST_MODE())
8861	{
8862	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8863	}
8864	#endif
8865	if (fLongModeGuest)
8866	{
8867	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8868	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8869	}
8870	else
8871	{
8872	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8873	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8874	VMX_IGS_TR_ATTR_TYPE_INVALID);
8875	}
8876	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8877	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8878	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8879	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8880	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8881	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8882	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8883	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8884
8885	/*
8886	* GDTR and IDTR.
8887	*/
8888	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8889	if (HMVMX_IS_64BIT_HOST_MODE())
8890	{
8891	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8892	AssertRCBreak(rc);
8893	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8894
8895	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8896	AssertRCBreak(rc);
8897	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8898	}
8899	#endif
8900
8901	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8902	AssertRCBreak(rc);
8903	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8904
8905	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8906	AssertRCBreak(rc);
8907	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8908
8909	/*
8910	* Guest Non-Register State.
8911	*/
8912	/* Activity State. */
8913	uint32_t u32ActivityState;
8914	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8915	AssertRCBreak(rc);
8916	HMVMX_CHECK_BREAK( !u32ActivityState
8917	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8918	VMX_IGS_ACTIVITY_STATE_INVALID);
8919	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8920	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8921	uint32_t u32IntrState;
8922	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8923	AssertRCBreak(rc);
8924	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8925	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8926	{
8927	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8928	}
8929
8930	/** @todo Activity state and injecting interrupts. Left as a todo since we
8931	* currently don't use activity states but ACTIVE. */
8932
8933	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8934	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8935
8936	/* Guest interruptibility-state. */
8937	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8938	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8939	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8940	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8941	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8942	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8943	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8944	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8945	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8946	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
8947	{
8948	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8949	{
8950	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8951	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8952	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8953	}
8954	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8955	{
8956	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8957	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8958	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8959	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8960	}
8961	}
8962	/** @todo Assumes the processor is not in SMM. */
8963	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8964	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8965	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8966	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8967	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8968	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8969	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8970	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8971	{
8972	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8973	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8974	}
8975
8976	/* Pending debug exceptions. */
8977	if (HMVMX_IS_64BIT_HOST_MODE())
8978	{
8979	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8980	AssertRCBreak(rc);
8981	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8982	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8983	u32Val = u64Val; /* For pending debug exceptions checks below. */
8984	}
8985	else
8986	{
8987	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8988	AssertRCBreak(rc);
8989	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8990	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8991	}
8992
8993	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8994	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8995	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8996	{
8997	if ( (u32Eflags & X86_EFL_TF)
8998	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8999	{
9000	/* Bit 14 is PendingDebug.BS. */
9001	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9002	}
9003	if ( !(u32Eflags & X86_EFL_TF)
9004	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9005	{
9006	/* Bit 14 is PendingDebug.BS. */
9007	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9008	}
9009	}
9010
9011	/* VMCS link pointer. */
9012	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9013	AssertRCBreak(rc);
9014	if (u64Val != UINT64_C(0xffffffffffffffff))
9015	{
9016	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9017	/** @todo Bits beyond the processor's physical-address width MBZ. */
9018	/** @todo 32-bit located in memory referenced by value of this field (as a
9019	* physical address) must contain the processor's VMCS revision ID. */
9020	/** @todo SMM checks. */
9021	}
9022
9023	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
9024
9025	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9026	if (uError == VMX_IGS_ERROR)
9027	uError = VMX_IGS_REASON_NOT_FOUND;
9028	} while (0);
9029
9030	pVCpu->hm.s.u32HMError = uError;
9031	return uError;
9032
9033	#undef HMVMX_ERROR_BREAK
9034	#undef HMVMX_CHECK_BREAK
9035	#undef HMVMX_IS_CANONICAL
9036	}
9037
9038	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9039	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9040	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9041
9042	/** @name VM-exit handlers.
9043	* @{
9044	*/
9045
9046	/**
9047	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9048	*/
9049	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9050	{
9051	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9053	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9054	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9055	return VINF_SUCCESS;
9056	return VINF_EM_RAW_INTERRUPT;
9057	}
9058
9059
9060	/**
9061	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9062	*/
9063	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9064	{
9065	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9066	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9067
9068	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9069	AssertRCReturn(rc, rc);
9070
9071	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9072	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9073	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9074	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9075
9076	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9077	{
9078	/*
9079	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9080	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9081	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9082	*
9083	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9084	*/
9085	VMXDispatchHostNmi();
9086	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9087	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9088	return VINF_SUCCESS;
9089	}
9090
9091	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9092	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9093	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9094	{
9095	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9096	return VINF_SUCCESS;
9097	}
9098	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9099	{
9100	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9101	return rc;
9102	}
9103
9104	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9105	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9106	switch (uIntType)
9107	{
9108	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9109	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9110	/* no break */
9111	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9112	{
9113	switch (uVector)
9114	{
9115	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9116	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9117	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9118	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9119	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9120	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9121	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9122	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9123	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9124	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9125	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9126	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9127	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9128	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9129	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9130	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9131	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9132	#endif
9133	default:
9134	{
9135	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9136	AssertRCReturn(rc, rc);
9137
9138	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9139	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9140	{
9141	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9142	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9143	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9144
9145	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9146	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9147	AssertRCReturn(rc, rc);
9148	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9149	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9150	0 /* GCPtrFaultAddress */);
9151	AssertRCReturn(rc, rc);
9152	}
9153	else
9154	{
9155	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9156	pVCpu->hm.s.u32HMError = uVector;
9157	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9158	}
9159	break;
9160	}
9161	}
9162	break;
9163	}
9164
9165	default:
9166	{
9167	pVCpu->hm.s.u32HMError = uExitIntInfo;
9168	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9169	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9170	break;
9171	}
9172	}
9173	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9174	return rc;
9175	}
9176
9177
9178	/**
9179	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9180	*/
9181	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9182	{
9183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9184
9185	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9186	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9187	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9188	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9189	AssertRCReturn(rc, rc);
9190
9191	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9193	return VINF_SUCCESS;
9194	}
9195
9196
9197	/**
9198	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9199	*/
9200	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9201	{
9202	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9203	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9204	HMVMX_RETURN_UNEXPECTED_EXIT();
9205	}
9206
9207
9208	/**
9209	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9210	*/
9211	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9212	{
9213	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9214	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9215	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9216	}
9217
9218
9219	/**
9220	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9221	*/
9222	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9223	{
9224	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9226	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9227	}
9228
9229
9230	/**
9231	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9232	*/
9233	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9234	{
9235	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9236	PVM pVM = pVCpu->CTX_SUFF(pVM);
9237	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9238	if (RT_LIKELY(rc == VINF_SUCCESS))
9239	{
9240	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9241	Assert(pVmxTransient->cbInstr == 2);
9242	}
9243	else
9244	{
9245	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9246	rc = VERR_EM_INTERPRETER;
9247	}
9248	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9249	return rc;
9250	}
9251
9252
9253	/**
9254	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9255	*/
9256	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9257	{
9258	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9259	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9260	AssertRCReturn(rc, rc);
9261
9262	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9263	return VINF_EM_RAW_EMULATE_INSTR;
9264
9265	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9266	HMVMX_RETURN_UNEXPECTED_EXIT();
9267	}
9268
9269
9270	/**
9271	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9272	*/
9273	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9274	{
9275	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9276	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9277	AssertRCReturn(rc, rc);
9278
9279	PVM pVM = pVCpu->CTX_SUFF(pVM);
9280	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9281	if (RT_LIKELY(rc == VINF_SUCCESS))
9282	{
9283	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9284	Assert(pVmxTransient->cbInstr == 2);
9285	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9286	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9287	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9288	}
9289	else
9290	{
9291	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9292	rc = VERR_EM_INTERPRETER;
9293	}
9294	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9295	return rc;
9296	}
9297
9298
9299	/**
9300	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9301	*/
9302	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9303	{
9304	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9305	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9306	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9307	AssertRCReturn(rc, rc);
9308
9309	PVM pVM = pVCpu->CTX_SUFF(pVM);
9310	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9311	if (RT_LIKELY(rc == VINF_SUCCESS))
9312	{
9313	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9314	Assert(pVmxTransient->cbInstr == 3);
9315	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9316	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9317	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9318	}
9319	else
9320	{
9321	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9322	rc = VERR_EM_INTERPRETER;
9323	}
9324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9325	return rc;
9326	}
9327
9328
9329	/**
9330	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9331	*/
9332	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9333	{
9334	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9335	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9336	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9337	AssertRCReturn(rc, rc);
9338
9339	PVM pVM = pVCpu->CTX_SUFF(pVM);
9340	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9341	if (RT_LIKELY(rc == VINF_SUCCESS))
9342	{
9343	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9344	Assert(pVmxTransient->cbInstr == 2);
9345	}
9346	else
9347	{
9348	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9349	rc = VERR_EM_INTERPRETER;
9350	}
9351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9352	return rc;
9353	}
9354
9355
9356	/**
9357	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9358	*/
9359	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9360	{
9361	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9362	PVM pVM = pVCpu->CTX_SUFF(pVM);
9363	Assert(!pVM->hm.s.fNestedPaging);
9364
9365	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
9366	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9367	AssertRCReturn(rc, rc);
9368
9369	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9370	rc = VBOXSTRICTRC_VAL(rc2);
9371	if (RT_LIKELY(rc == VINF_SUCCESS))
9372	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9373	else
9374	{
9375	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9376	pVmxTransient->uExitQualification, rc));
9377	}
9378	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9379	return rc;
9380	}
9381
9382
9383	/**
9384	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9385	*/
9386	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9387	{
9388	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9389	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9390	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9391	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9392	AssertRCReturn(rc, rc);
9393
9394	PVM pVM = pVCpu->CTX_SUFF(pVM);
9395	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9396	if (RT_LIKELY(rc == VINF_SUCCESS))
9397	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9398	else
9399	{
9400	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9401	rc = VERR_EM_INTERPRETER;
9402	}
9403	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9404	return rc;
9405	}
9406
9407
9408	/**
9409	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9410	*/
9411	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9412	{
9413	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9414	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9415	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9416	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9417	AssertRCReturn(rc, rc);
9418
9419	PVM pVM = pVCpu->CTX_SUFF(pVM);
9420	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9421	rc = VBOXSTRICTRC_VAL(rc2);
9422	if (RT_LIKELY( rc == VINF_SUCCESS
9423	\|\| rc == VINF_EM_HALT))
9424	{
9425	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9426	AssertRCReturn(rc3, rc3);
9427
9428	if ( rc == VINF_EM_HALT
9429	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9430	{
9431	rc = VINF_SUCCESS;
9432	}
9433	}
9434	else
9435	{
9436	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9437	rc = VERR_EM_INTERPRETER;
9438	}
9439	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9440	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9441	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9442	return rc;
9443	}
9444
9445
9446	/**
9447	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9448	*/
9449	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9450	{
9451	/*
9452	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9453	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9454	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9455	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9456	*/
9457	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9458	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9459	HMVMX_RETURN_UNEXPECTED_EXIT();
9460	}
9461
9462
9463	/**
9464	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9465	*/
9466	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9467	{
9468	/*
9469	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9470	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9471	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9472	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9473	*/
9474	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9475	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9476	HMVMX_RETURN_UNEXPECTED_EXIT();
9477	}
9478
9479
9480	/**
9481	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9482	*/
9483	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9484	{
9485	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9486	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9487	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9488	HMVMX_RETURN_UNEXPECTED_EXIT();
9489	}
9490
9491
9492	/**
9493	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9494	*/
9495	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9496	{
9497	/*
9498	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9499	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9500	* See Intel spec. 25.3 "Other Causes of VM-exits".
9501	*/
9502	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9503	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9504	HMVMX_RETURN_UNEXPECTED_EXIT();
9505	}
9506
9507
9508	/**
9509	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9510	* VM-exit.
9511	*/
9512	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9513	{
9514	/*
9515	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9516	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9517	*
9518	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9519	* See Intel spec. "23.8 Restrictions on VMX operation".
9520	*/
9521	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9522	return VINF_SUCCESS;
9523	}
9524
9525
9526	/**
9527	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9528	* VM-exit.
9529	*/
9530	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9531	{
9532	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9533	return VINF_EM_RESET;
9534	}
9535
9536
9537	/**
9538	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9539	*/
9540	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9541	{
9542	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9543	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9544	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9545	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9546	AssertRCReturn(rc, rc);
9547
9548	pMixedCtx->rip++;
9549	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9550	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9551	rc = VINF_SUCCESS;
9552	else
9553	rc = VINF_EM_HALT;
9554
9555	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9556	return rc;
9557	}
9558
9559
9560	/**
9561	* VM-exit handler for instructions that result in a #UD exception delivered to
9562	* the guest.
9563	*/
9564	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9565	{
9566	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9567	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9568	return VINF_SUCCESS;
9569	}
9570
9571
9572	/**
9573	* VM-exit handler for expiry of the VMX preemption timer.
9574	*/
9575	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9576	{
9577	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9578
9579	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9580	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9581
9582	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9583	PVM pVM = pVCpu->CTX_SUFF(pVM);
9584	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9586	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9587	}
9588
9589
9590	/**
9591	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9592	*/
9593	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9594	{
9595	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9596
9597	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9598	/** @todo check if XSETBV is supported by the recompiler. */
9599	return VERR_EM_INTERPRETER;
9600	}
9601
9602
9603	/**
9604	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9605	*/
9606	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9607	{
9608	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9609
9610	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9611	/** @todo implement EMInterpretInvpcid() */
9612	return VERR_EM_INTERPRETER;
9613	}
9614
9615
9616	/**
9617	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9618	* Error VM-exit.
9619	*/
9620	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9621	{
9622	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9623	AssertRCReturn(rc, rc);
9624
9625	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9626	NOREF(uInvalidReason);
9627
9628	#ifdef VBOX_STRICT
9629	uint32_t uIntrState;
9630	HMVMXHCUINTREG uHCReg;
9631	uint64_t u64Val;
9632	uint32_t u32Val;
9633
9634	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9635	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9636	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9637	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9638	AssertRCReturn(rc, rc);
9639
9640	Log4(("uInvalidReason %u\n", uInvalidReason));
9641	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9642	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9643	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9644	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9645
9646	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9647	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9648	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9649	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9650	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9651	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9652	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9653	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9654	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9655	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9656	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9657	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9658	#else
9659	NOREF(pVmxTransient);
9660	#endif
9661
9662	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9663	return VERR_VMX_INVALID_GUEST_STATE;
9664	}
9665
9666
9667	/**
9668	* VM-exit handler for VM-entry failure due to an MSR-load
9669	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9670	*/
9671	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9672	{
9673	NOREF(pVmxTransient);
9674	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9675	HMVMX_RETURN_UNEXPECTED_EXIT();
9676	}
9677
9678
9679	/**
9680	* VM-exit handler for VM-entry failure due to a machine-check event
9681	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9682	*/
9683	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9684	{
9685	NOREF(pVmxTransient);
9686	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9687	HMVMX_RETURN_UNEXPECTED_EXIT();
9688	}
9689
9690
9691	/**
9692	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9693	* theory.
9694	*/
9695	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9696	{
9697	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9698	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9699	return VERR_VMX_UNDEFINED_EXIT_CODE;
9700	}
9701
9702
9703	/**
9704	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9705	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9706	* Conditional VM-exit.
9707	*/
9708	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9709	{
9710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9711
9712	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9714	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9715	return VERR_EM_INTERPRETER;
9716	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9717	HMVMX_RETURN_UNEXPECTED_EXIT();
9718	}
9719
9720
9721	/**
9722	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9723	*/
9724	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9725	{
9726	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9727
9728	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9730	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9731	return VERR_EM_INTERPRETER;
9732	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9733	HMVMX_RETURN_UNEXPECTED_EXIT();
9734	}
9735
9736
9737	/**
9738	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9739	*/
9740	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9741	{
9742	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9743
9744	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9745	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9746	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9747	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9748	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9749	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9750	AssertRCReturn(rc, rc);
9751	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9752
9753	#ifdef VBOX_STRICT
9754	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9755	&& hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9756	{
9757	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9758	HMVMX_RETURN_UNEXPECTED_EXIT();
9759	}
9760	#endif
9761
9762	PVM pVM = pVCpu->CTX_SUFF(pVM);
9763	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9764	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9765	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9766	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9767
9768	if (RT_LIKELY(rc == VINF_SUCCESS))
9769	{
9770	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9771	Assert(pVmxTransient->cbInstr == 2);
9772	}
9773	return rc;
9774	}
9775
9776
9777	/**
9778	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9779	*/
9780	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9781	{
9782	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9783	PVM pVM = pVCpu->CTX_SUFF(pVM);
9784	int rc = VINF_SUCCESS;
9785
9786	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9787	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9788	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9789	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9790	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9791	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9792	AssertRCReturn(rc, rc);
9793	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9794
9795	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9796	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9797	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9798
9799	if (RT_LIKELY(rc == VINF_SUCCESS))
9800	{
9801	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9802
9803	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9804	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9805	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9806	{
9807	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9808	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9809	EMInterpretWrmsr() changes it. */
9810	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9811	}
9812	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9813	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9814
9815	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
9816	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9817	{
9818	switch (pMixedCtx->ecx)
9819	{
9820	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9821	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9822	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9823	case MSR_K8_FS_BASE: /* no break */
9824	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9825	default:
9826	{
9827	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9828	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9829	break;
9830	}
9831	}
9832	}
9833	#ifdef VBOX_STRICT
9834	else
9835	{
9836	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9837	switch (pMixedCtx->ecx)
9838	{
9839	case MSR_IA32_SYSENTER_CS:
9840	case MSR_IA32_SYSENTER_EIP:
9841	case MSR_IA32_SYSENTER_ESP:
9842	case MSR_K8_FS_BASE:
9843	case MSR_K8_GS_BASE:
9844	{
9845	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9846	HMVMX_RETURN_UNEXPECTED_EXIT();
9847	}
9848
9849	/* Writes to MSRs that are part of the auto-load/store are shouldn't cause VM-exits
9850	when MSR-bitmaps are supported. */
9851	default:
9852	{
9853	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9854	{
9855	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9856	pMixedCtx->ecx));
9857	HMVMX_RETURN_UNEXPECTED_EXIT();
9858	}
9859	break;
9860	}
9861	}
9862	}
9863	#endif /* VBOX_STRICT */
9864	}
9865	return rc;
9866	}
9867
9868
9869	/**
9870	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9871	*/
9872	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9873	{
9874	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9875
9876	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9877	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9878	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9879	return VERR_EM_INTERPRETER;
9880	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9881	HMVMX_RETURN_UNEXPECTED_EXIT();
9882	}
9883
9884
9885	/**
9886	* VM-exit handler for when the TPR value is lowered below the specified
9887	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9888	*/
9889	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9890	{
9891	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9892	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9893
9894	/*
9895	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9896	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
9897	* resume guest execution.
9898	*/
9899	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9900	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9901	return VINF_SUCCESS;
9902	}
9903
9904
9905	/**
9906	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9907	* VM-exit.
9908	*
9909	* @retval VINF_SUCCESS when guest execution can continue.
9910	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9911	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9912	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9913	* recompiler.
9914	*/
9915	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9916	{
9917	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9918	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9919	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
9920	AssertRCReturn(rc, rc);
9921
9922	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9923	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9924	PVM pVM = pVCpu->CTX_SUFF(pVM);
9925	switch (uAccessType)
9926	{
9927	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9928	{
9929	#if 0
9930	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9931	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9932	#else
9933	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9934	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9935	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9936	#endif
9937	AssertRCReturn(rc, rc);
9938
9939	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9940	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9941	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9942	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9943
9944	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9945	{
9946	case 0: /* CR0 */
9947	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9948	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9949	break;
9950	case 2: /* CR2 */
9951	/* Nothing to do here, CR2 it's not part of the VMCS. */
9952	break;
9953	case 3: /* CR3 */
9954	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9955	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9956	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9957	break;
9958	case 4: /* CR4 */
9959	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9960	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9961	break;
9962	case 8: /* CR8 */
9963	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9964	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9965	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9966	break;
9967	default:
9968	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9969	break;
9970	}
9971
9972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9973	break;
9974	}
9975
9976	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9977	{
9978	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9979	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9980	AssertRCReturn(rc, rc);
9981	Assert( !pVM->hm.s.fNestedPaging
9982	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9983	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9984
9985	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9986	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9987	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9988
9989	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9990	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9991	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9992	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9993	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9994	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9995	break;
9996	}
9997
9998	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9999	{
10000	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10001	AssertRCReturn(rc, rc);
10002	rc = EMInterpretCLTS(pVM, pVCpu);
10003	AssertRCReturn(rc, rc);
10004	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10005	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10006	Log4(("CRX CLTS write rc=%d\n", rc));
10007	break;
10008	}
10009
10010	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10011	{
10012	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10013	AssertRCReturn(rc, rc);
10014	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10015	if (RT_LIKELY(rc == VINF_SUCCESS))
10016	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10018	Log4(("CRX LMSW write rc=%d\n", rc));
10019	break;
10020	}
10021
10022	default:
10023	{
10024	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10025	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10026	}
10027	}
10028
10029	/* Validate possible error codes. */
10030	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10031	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10032	if (RT_SUCCESS(rc))
10033	{
10034	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10035	AssertRCReturn(rc2, rc2);
10036	}
10037
10038	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10039	return rc;
10040	}
10041
10042
10043	/**
10044	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10045	* VM-exit.
10046	*/
10047	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10048	{
10049	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10050	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10051
10052	int rc2 = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
10053	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10054	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10055	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10056	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10057	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10058	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10059	AssertRCReturn(rc2, rc2);
10060
10061	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10062	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10063	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10064	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10065	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10066	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10067	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10068
10069	/* I/O operation lookup arrays. */
10070	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10071	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10072
10073	VBOXSTRICTRC rcStrict;
10074	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10075	const uint32_t cbInstr = pVmxTransient->cbInstr;
10076	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10077	PVM pVM = pVCpu->CTX_SUFF(pVM);
10078	if (fIOString)
10079	{
10080	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
10081	/*
10082	* INS/OUTS - I/O String instruction.
10083	*
10084	* Use instruction-information if available, otherwise fall back on
10085	* interpreting the instruction.
10086	*/
10087	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10088	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10089	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10090	{
10091	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10092	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10093	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10094	AssertRCReturn(rc2, rc2);
10095	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10096	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10097	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10098	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10099	if (fIOWrite)
10100	{
10101	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10102	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10103	}
10104	else
10105	{
10106	/*
10107	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10108	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10109	* See Intel Instruction spec. for "INS".
10110	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10111	*/
10112	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10113	}
10114	}
10115	else
10116	{
10117	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10118	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10119	AssertRCReturn(rc2, rc2);
10120	rcStrict = IEMExecOne(pVCpu);
10121	}
10122	/** @todo IEM needs to be setting these flags somehow. */
10123	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10124	fUpdateRipAlready = true;
10125	#else
10126	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10127	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10128	if (RT_SUCCESS(rcStrict))
10129	{
10130	if (fIOWrite)
10131	{
10132	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10133	(DISCPUMODE)pDis->uAddrMode, cbValue);
10134	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10135	}
10136	else
10137	{
10138	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10139	(DISCPUMODE)pDis->uAddrMode, cbValue);
10140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10141	}
10142	}
10143	else
10144	{
10145	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10146	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10147	}
10148	#endif
10149	}
10150	else
10151	{
10152	/*
10153	* IN/OUT - I/O instruction.
10154	*/
10155	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10156	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10157	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10158	if (fIOWrite)
10159	{
10160	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10161	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10162	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10163	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10164	}
10165	else
10166	{
10167	uint32_t u32Result = 0;
10168	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10169	if (IOM_SUCCESS(rcStrict))
10170	{
10171	/* Save result of I/O IN instr. in AL/AX/EAX. */
10172	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10173	}
10174	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10175	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10176	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10177	}
10178	}
10179
10180	if (IOM_SUCCESS(rcStrict))
10181	{
10182	if (!fUpdateRipAlready)
10183	{
10184	pMixedCtx->rip += cbInstr;
10185	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10186	}
10187
10188	/*
10189	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10190	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10191	*/
10192	if (fIOString)
10193	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10194
10195	/*
10196	* If any I/O breakpoints are armed, we need to check if one triggered
10197	* and take appropriate action.
10198	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10199	*/
10200	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10201	AssertRCReturn(rc2, rc2);
10202
10203	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10204	* execution engines about whether hyper BPs and such are pending. */
10205	uint32_t const uDr7 = pMixedCtx->dr[7];
10206	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10207	&& X86_DR7_ANY_RW_IO(uDr7)
10208	&& (pMixedCtx->cr4 & X86_CR4_DE))
10209	\|\| DBGFBpIsHwIoArmed(pVM)))
10210	{
10211	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10212
10213	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10214	VMMRZCallRing3Disable(pVCpu);
10215	HM_DISABLE_PREEMPT_IF_NEEDED();
10216
10217	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10218
10219	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10220	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10221	{
10222	/* Raise #DB. */
10223	if (fIsGuestDbgActive)
10224	ASMSetDR6(pMixedCtx->dr[6]);
10225	if (pMixedCtx->dr[7] != uDr7)
10226	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10227
10228	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10229	}
10230	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10231	else if ( rcStrict2 != VINF_SUCCESS
10232	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10233	rcStrict = rcStrict2;
10234
10235	HM_RESTORE_PREEMPT_IF_NEEDED();
10236	VMMRZCallRing3Enable(pVCpu);
10237	}
10238	}
10239
10240	#ifdef DEBUG
10241	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10242	Assert(!fIOWrite);
10243	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10244	Assert(fIOWrite);
10245	else
10246	{
10247	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10248	* statuses, that the VMM device and some others may return. See
10249	* IOM_SUCCESS() for guidance. */
10250	AssertMsg( RT_FAILURE(rcStrict)
10251	\|\| rcStrict == VINF_SUCCESS
10252	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10253	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10254	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10255	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10256	}
10257	#endif
10258
10259	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10260	return VBOXSTRICTRC_TODO(rcStrict);
10261	}
10262
10263
10264	/**
10265	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10266	* VM-exit.
10267	*/
10268	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10269	{
10270	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10271
10272	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10273	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
10274	AssertRCReturn(rc, rc);
10275	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10276	{
10277	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10278	AssertRCReturn(rc, rc);
10279	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10280	{
10281	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10282
10283	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10284	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10285	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10286	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10287	{
10288	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10289	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10290
10291	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10292	Assert(!pVCpu->hm.s.Event.fPending);
10293	pVCpu->hm.s.Event.fPending = true;
10294	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10295	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10296	AssertRCReturn(rc, rc);
10297	if (fErrorCodeValid)
10298	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10299	else
10300	pVCpu->hm.s.Event.u32ErrCode = 0;
10301	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10302	&& uVector == X86_XCPT_PF)
10303	{
10304	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10305	}
10306
10307	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10308	}
10309	}
10310	}
10311
10312	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10313	* emulation. */
10314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10315	return VERR_EM_INTERPRETER;
10316	}
10317
10318
10319	/**
10320	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10321	*/
10322	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10323	{
10324	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10325	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10326	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10327	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10328	AssertRCReturn(rc, rc);
10329	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10330	return VINF_EM_DBG_STEPPED;
10331	}
10332
10333
10334	/**
10335	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10336	*/
10337	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10338	{
10339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10340
10341	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10342	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10343	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10344	return VINF_SUCCESS;
10345	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10346	return rc;
10347
10348	#if 0
10349	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10350	* just sync the whole thing. */
10351	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10352	#else
10353	/* Aggressive state sync. for now. */
10354	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10355	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10356	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10357	#endif
10358	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
10359	AssertRCReturn(rc, rc);
10360
10361	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10362	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10363	switch (uAccessType)
10364	{
10365	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10366	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10367	{
10368	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10369	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10370	{
10371	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10372	}
10373
10374	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10375	GCPhys &= PAGE_BASE_GC_MASK;
10376	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10377	PVM pVM = pVCpu->CTX_SUFF(pVM);
10378	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10379	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10380
10381	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10382	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10383	CPUMCTX2CORE(pMixedCtx), GCPhys);
10384	rc = VBOXSTRICTRC_VAL(rc2);
10385	Log4(("ApicAccess rc=%d\n", rc));
10386	if ( rc == VINF_SUCCESS
10387	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10388	\|\| rc == VERR_PAGE_NOT_PRESENT)
10389	{
10390	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10391	\| HM_CHANGED_GUEST_RSP
10392	\| HM_CHANGED_GUEST_RFLAGS
10393	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10394	rc = VINF_SUCCESS;
10395	}
10396	break;
10397	}
10398
10399	default:
10400	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10401	rc = VINF_EM_RAW_EMULATE_INSTR;
10402	break;
10403	}
10404
10405	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10406	return rc;
10407	}
10408
10409
10410	/**
10411	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10412	* VM-exit.
10413	*/
10414	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10415	{
10416	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10417
10418	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10419	if (pVmxTransient->fWasGuestDebugStateActive)
10420	{
10421	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10422	HMVMX_RETURN_UNEXPECTED_EXIT();
10423	}
10424
10425	int rc = VERR_INTERNAL_ERROR_5;
10426	if ( !DBGFIsStepping(pVCpu)
10427	&& !pVCpu->hm.s.fSingleInstruction
10428	&& !pVmxTransient->fWasHyperDebugStateActive)
10429	{
10430	/* Don't intercept MOV DRx and #DB any more. */
10431	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10432	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10433	AssertRCReturn(rc, rc);
10434
10435	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10436	{
10437	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10438	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10439	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10440	AssertRCReturn(rc, rc);
10441	#endif
10442	}
10443
10444	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10445	VMMRZCallRing3Disable(pVCpu);
10446	HM_DISABLE_PREEMPT_IF_NEEDED();
10447
10448	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10449	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10450	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10451
10452	HM_RESTORE_PREEMPT_IF_NEEDED();
10453	VMMRZCallRing3Enable(pVCpu);
10454
10455	#ifdef VBOX_WITH_STATISTICS
10456	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
10457	AssertRCReturn(rc, rc);
10458	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10459	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10460	else
10461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10462	#endif
10463	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10464	return VINF_SUCCESS;
10465	}
10466
10467	/*
10468	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10469	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update the segment registers and DR7 from the CPU.
10470	*/
10471	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
10472	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10473	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10474	AssertRCReturn(rc, rc);
10475	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10476
10477	PVM pVM = pVCpu->CTX_SUFF(pVM);
10478	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10479	{
10480	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10481	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10482	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10483	if (RT_SUCCESS(rc))
10484	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10486	}
10487	else
10488	{
10489	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10490	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10491	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10492	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10493	}
10494
10495	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10496	if (RT_SUCCESS(rc))
10497	{
10498	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10499	AssertRCReturn(rc2, rc2);
10500	}
10501	return rc;
10502	}
10503
10504
10505	/**
10506	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10507	* Conditional VM-exit.
10508	*/
10509	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10510	{
10511	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10512	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10513
10514	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10515	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10516	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10517	return VINF_SUCCESS;
10518	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10519	return rc;
10520
10521	RTGCPHYS GCPhys = 0;
10522	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10523
10524	#if 0
10525	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10526	#else
10527	/* Aggressive state sync. for now. */
10528	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10529	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10530	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10531	#endif
10532	AssertRCReturn(rc, rc);
10533
10534	/*
10535	* If we succeed, resume guest execution.
10536	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10537	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10538	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10539	* weird case. See @bugref{6043}.
10540	*/
10541	PVM pVM = pVCpu->CTX_SUFF(pVM);
10542	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10543	rc = VBOXSTRICTRC_VAL(rc2);
10544	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10545	if ( rc == VINF_SUCCESS
10546	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10547	\|\| rc == VERR_PAGE_NOT_PRESENT)
10548	{
10549	/* Successfully handled MMIO operation. */
10550	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10551	\| HM_CHANGED_GUEST_RSP
10552	\| HM_CHANGED_GUEST_RFLAGS
10553	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10554	rc = VINF_SUCCESS;
10555	}
10556	return rc;
10557	}
10558
10559
10560	/**
10561	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10562	* VM-exit.
10563	*/
10564	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10565	{
10566	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10567	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10568
10569	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10570	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10571	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10572	return VINF_SUCCESS;
10573	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10574	return rc;
10575
10576	RTGCPHYS GCPhys = 0;
10577	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10578	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
10579	#if 0
10580	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10581	#else
10582	/* Aggressive state sync. for now. */
10583	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10584	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10585	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10586	#endif
10587	AssertRCReturn(rc, rc);
10588
10589	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10590	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10591
10592	RTGCUINT uErrorCode = 0;
10593	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10594	uErrorCode \|= X86_TRAP_PF_ID;
10595	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10596	uErrorCode \|= X86_TRAP_PF_RW;
10597	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10598	uErrorCode \|= X86_TRAP_PF_P;
10599
10600	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10601
10602	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10603	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10604
10605	/* Handle the pagefault trap for the nested shadow table. */
10606	PVM pVM = pVCpu->CTX_SUFF(pVM);
10607	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10608	TRPMResetTrap(pVCpu);
10609
10610	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10611	if ( rc == VINF_SUCCESS
10612	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10613	\|\| rc == VERR_PAGE_NOT_PRESENT)
10614	{
10615	/* Successfully synced our nested page tables. */
10616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10617	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10618	\| HM_CHANGED_GUEST_RSP
10619	\| HM_CHANGED_GUEST_RFLAGS);
10620	return VINF_SUCCESS;
10621	}
10622
10623	Log4(("EPT return to ring-3 rc=%d\n"));
10624	return rc;
10625	}
10626
10627	/** @} */
10628
10629	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10630	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10631	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10632
10633	/** @name VM-exit exception handlers.
10634	* @{
10635	*/
10636
10637	/**
10638	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10639	*/
10640	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10641	{
10642	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10643	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10644
10645	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10646	AssertRCReturn(rc, rc);
10647
10648	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10649	{
10650	/* Old-style FPU error reporting needs some extra work. */
10651	/** @todo don't fall back to the recompiler, but do it manually. */
10652	return VERR_EM_INTERPRETER;
10653	}
10654
10655	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10656	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10657	return rc;
10658	}
10659
10660
10661	/**
10662	* VM-exit exception handler for #BP (Breakpoint exception).
10663	*/
10664	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10665	{
10666	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10668
10669	/** @todo Try optimize this by not saving the entire guest state unless
10670	* really needed. */
10671	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10672	AssertRCReturn(rc, rc);
10673
10674	PVM pVM = pVCpu->CTX_SUFF(pVM);
10675	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10676	if (rc == VINF_EM_RAW_GUEST_TRAP)
10677	{
10678	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10679	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10680	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10681	AssertRCReturn(rc, rc);
10682
10683	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10684	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10685	}
10686
10687	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10688	return rc;
10689	}
10690
10691
10692	/**
10693	* VM-exit exception handler for #DB (Debug exception).
10694	*/
10695	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10696	{
10697	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10698	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10699	Log6(("XcptDB\n"));
10700
10701	/*
10702	* Get the DR6-like values from the exit qualification and pass it to DBGF
10703	* for processing.
10704	*/
10705	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
10706	AssertRCReturn(rc, rc);
10707
10708	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10709	uint64_t uDR6 = X86_DR6_INIT_VAL;
10710	uDR6 \|= ( pVmxTransient->uExitQualification
10711	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10712
10713	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10714	if (rc == VINF_EM_RAW_GUEST_TRAP)
10715	{
10716	/*
10717	* The exception was for the guest. Update DR6, DR7.GD and
10718	* IA32_DEBUGCTL.LBR before forwarding it.
10719	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10720	*/
10721	VMMRZCallRing3Disable(pVCpu);
10722	HM_DISABLE_PREEMPT_IF_NEEDED();
10723
10724	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10725	pMixedCtx->dr[6] \|= uDR6;
10726	if (CPUMIsGuestDebugStateActive(pVCpu))
10727	ASMSetDR6(pMixedCtx->dr[6]);
10728
10729	HM_RESTORE_PREEMPT_IF_NEEDED();
10730	VMMRZCallRing3Enable(pVCpu);
10731
10732	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10733	AssertRCReturn(rc, rc);
10734
10735	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10736	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10737
10738	/* Paranoia. */
10739	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10740	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10741
10742	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10743	AssertRCReturn(rc, rc);
10744
10745	/*
10746	* Raise #DB in the guest.
10747	*/
10748	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10749	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10750	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10751	AssertRCReturn(rc, rc);
10752	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10753	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10754	return VINF_SUCCESS;
10755	}
10756
10757	/*
10758	* Not a guest trap, must be a hypervisor related debug event then.
10759	* Update DR6 in case someone is interested in it.
10760	*/
10761	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10762	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10763	CPUMSetHyperDR6(pVCpu, uDR6);
10764
10765	return rc;
10766	}
10767
10768
10769	/**
10770	* VM-exit exception handler for #NM (Device-not-available exception: floating
10771	* point exception).
10772	*/
10773	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10774	{
10775	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10776
10777	/* We require CR0 and EFER. EFER is always up-to-date. */
10778	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10779	AssertRCReturn(rc, rc);
10780
10781	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10782	VMMRZCallRing3Disable(pVCpu);
10783	HM_DISABLE_PREEMPT_IF_NEEDED();
10784
10785	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10786	if (pVmxTransient->fWasGuestFPUStateActive)
10787	{
10788	rc = VINF_EM_RAW_GUEST_TRAP;
10789	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10790	}
10791	else
10792	{
10793	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10794	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10795	#endif
10796	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10797	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10798	}
10799
10800	HM_RESTORE_PREEMPT_IF_NEEDED();
10801	VMMRZCallRing3Enable(pVCpu);
10802
10803	if (rc == VINF_SUCCESS)
10804	{
10805	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10806	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10807	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10808	pVCpu->hm.s.fUseGuestFpu = true;
10809	}
10810	else
10811	{
10812	/* Forward #NM to the guest. */
10813	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10814	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10815	AssertRCReturn(rc, rc);
10816	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10817	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
10818	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10819	}
10820
10821	return VINF_SUCCESS;
10822	}
10823
10824
10825	/**
10826	* VM-exit exception handler for #GP (General-protection exception).
10827	*
10828	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10829	*/
10830	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10831	{
10832	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10833	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10834
10835	int rc = VERR_INTERNAL_ERROR_5;
10836	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10837	{
10838	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10839	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10840	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10841	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10842	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10843	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10844	AssertRCReturn(rc, rc);
10845	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10846	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10847	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10848	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10849	return rc;
10850	#else
10851	/* We don't intercept #GP. */
10852	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10853	NOREF(pVmxTransient);
10854	return VERR_VMX_UNEXPECTED_EXCEPTION;
10855	#endif
10856	}
10857
10858	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10859	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10860
10861	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10862	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10863	AssertRCReturn(rc, rc);
10864
10865	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10866	uint32_t cbOp = 0;
10867	PVM pVM = pVCpu->CTX_SUFF(pVM);
10868	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10869	if (RT_SUCCESS(rc))
10870	{
10871	rc = VINF_SUCCESS;
10872	Assert(cbOp == pDis->cbInstr);
10873	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10874	switch (pDis->pCurInstr->uOpcode)
10875	{
10876	case OP_CLI:
10877	{
10878	pMixedCtx->eflags.Bits.u1IF = 0;
10879	pMixedCtx->rip += pDis->cbInstr;
10880	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10881	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10882	break;
10883	}
10884
10885	case OP_STI:
10886	{
10887	pMixedCtx->eflags.Bits.u1IF = 1;
10888	pMixedCtx->rip += pDis->cbInstr;
10889	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10890	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10891	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10892	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10893	break;
10894	}
10895
10896	case OP_HLT:
10897	{
10898	rc = VINF_EM_HALT;
10899	pMixedCtx->rip += pDis->cbInstr;
10900	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10901	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10902	break;
10903	}
10904
10905	case OP_POPF:
10906	{
10907	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10908	uint32_t cbParm = 0;
10909	uint32_t uMask = 0;
10910	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10911	{
10912	cbParm = 4;
10913	uMask = 0xffffffff;
10914	}
10915	else
10916	{
10917	cbParm = 2;
10918	uMask = 0xffff;
10919	}
10920
10921	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10922	RTGCPTR GCPtrStack = 0;
10923	X86EFLAGS Eflags;
10924	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10925	&GCPtrStack);
10926	if (RT_SUCCESS(rc))
10927	{
10928	Assert(sizeof(Eflags.u32) >= cbParm);
10929	Eflags.u32 = 0;
10930	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10931	}
10932	if (RT_FAILURE(rc))
10933	{
10934	rc = VERR_EM_INTERPRETER;
10935	break;
10936	}
10937	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10938	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10939	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10940	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10941	pMixedCtx->esp += cbParm;
10942	pMixedCtx->esp &= uMask;
10943	pMixedCtx->rip += pDis->cbInstr;
10944
10945	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10946	\| HM_CHANGED_GUEST_RSP
10947	\| HM_CHANGED_GUEST_RFLAGS);
10948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10949	break;
10950	}
10951
10952	case OP_PUSHF:
10953	{
10954	uint32_t cbParm = 0;
10955	uint32_t uMask = 0;
10956	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10957	{
10958	cbParm = 4;
10959	uMask = 0xffffffff;
10960	}
10961	else
10962	{
10963	cbParm = 2;
10964	uMask = 0xffff;
10965	}
10966
10967	/* Get the stack pointer & push the contents of eflags onto the stack. */
10968	RTGCPTR GCPtrStack = 0;
10969	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10970	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10971	if (RT_FAILURE(rc))
10972	{
10973	rc = VERR_EM_INTERPRETER;
10974	break;
10975	}
10976	X86EFLAGS Eflags = pMixedCtx->eflags;
10977	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10978	Eflags.Bits.u1RF = 0;
10979	Eflags.Bits.u1VM = 0;
10980
10981	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10982	if (RT_FAILURE(rc))
10983	{
10984	rc = VERR_EM_INTERPRETER;
10985	break;
10986	}
10987	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10988	pMixedCtx->esp -= cbParm;
10989	pMixedCtx->esp &= uMask;
10990	pMixedCtx->rip += pDis->cbInstr;
10991	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
10992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10993	break;
10994	}
10995
10996	case OP_IRET:
10997	{
10998	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10999	* instruction reference. */
11000	RTGCPTR GCPtrStack = 0;
11001	uint32_t uMask = 0xffff;
11002	uint16_t aIretFrame[3];
11003	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11004	{
11005	rc = VERR_EM_INTERPRETER;
11006	break;
11007	}
11008	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11009	&GCPtrStack);
11010	if (RT_SUCCESS(rc))
11011	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11012	if (RT_FAILURE(rc))
11013	{
11014	rc = VERR_EM_INTERPRETER;
11015	break;
11016	}
11017	pMixedCtx->eip = 0;
11018	pMixedCtx->ip = aIretFrame[0];
11019	pMixedCtx->cs.Sel = aIretFrame[1];
11020	pMixedCtx->cs.ValidSel = aIretFrame[1];
11021	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11022	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11023	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11024	pMixedCtx->sp += sizeof(aIretFrame);
11025	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11026	\| HM_CHANGED_GUEST_SEGMENT_REGS
11027	\| HM_CHANGED_GUEST_RSP
11028	\| HM_CHANGED_GUEST_RFLAGS);
11029	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11030	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11031	break;
11032	}
11033
11034	case OP_INT:
11035	{
11036	uint16_t uVector = pDis->Param1.uValue & 0xff;
11037	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11038	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11039	break;
11040	}
11041
11042	case OP_INTO:
11043	{
11044	if (pMixedCtx->eflags.Bits.u1OF)
11045	{
11046	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11047	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11048	}
11049	break;
11050	}
11051
11052	default:
11053	{
11054	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11055	EMCODETYPE_SUPERVISOR);
11056	rc = VBOXSTRICTRC_VAL(rc2);
11057	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11058	Log4(("#GP rc=%Rrc\n", rc));
11059	break;
11060	}
11061	}
11062	}
11063	else
11064	rc = VERR_EM_INTERPRETER;
11065
11066	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11067	("#GP Unexpected rc=%Rrc\n", rc));
11068	return rc;
11069	}
11070
11071
11072	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11073	/**
11074	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11075	* the exception reported in the VMX transient structure back into the VM.
11076	*
11077	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11078	* up-to-date.
11079	*/
11080	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11081	{
11082	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11083
11084	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11085	hmR0VmxCheckExitDueToEventDelivery(). */
11086	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11087	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11088	AssertRCReturn(rc, rc);
11089	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11090
11091	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11092	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11093	return VINF_SUCCESS;
11094	}
11095	#endif
11096
11097
11098	/**
11099	* VM-exit exception handler for #PF (Page-fault exception).
11100	*/
11101	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11102	{
11103	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11104	PVM pVM = pVCpu->CTX_SUFF(pVM);
11105	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
11106	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11107	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11108	AssertRCReturn(rc, rc);
11109
11110	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11111	if (pVM->hm.s.fNestedPaging)
11112	{
11113	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11114	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11115	{
11116	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11117	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11118	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11119	}
11120	else
11121	{
11122	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11123	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11124	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11125	}
11126	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11127	return rc;
11128	}
11129	#else
11130	Assert(!pVM->hm.s.fNestedPaging);
11131	NOREF(pVM);
11132	#endif
11133
11134	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11135	AssertRCReturn(rc, rc);
11136
11137	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11138	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11139
11140	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11141	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11142	(RTGCPTR)pVmxTransient->uExitQualification);
11143
11144	Log4(("#PF: rc=%Rrc\n", rc));
11145	if (rc == VINF_SUCCESS)
11146	{
11147	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11148	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11149	* memory? We don't update the whole state here... */
11150	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11151	\| HM_CHANGED_GUEST_RSP
11152	\| HM_CHANGED_GUEST_RFLAGS
11153	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11154	TRPMResetTrap(pVCpu);
11155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11156	return rc;
11157	}
11158	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11159	{
11160	if (!pVmxTransient->fVectoringPF)
11161	{
11162	/* It's a guest page fault and needs to be reflected to the guest. */
11163	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11164	TRPMResetTrap(pVCpu);
11165	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11166	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11167	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11168	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11169	}
11170	else
11171	{
11172	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11173	TRPMResetTrap(pVCpu);
11174	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11175	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11176	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11177	}
11178
11179	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11180	return VINF_SUCCESS;
11181	}
11182
11183	TRPMResetTrap(pVCpu);
11184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11185	return rc;
11186	}
11187
11188	/** @} */
11189

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49523

Download in other formats: