HMVMXR0.cpp@ 49966

Last change on this file since 49966 was 49903, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Todo comment.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 471.3 KB

Line
1	/* $Id: HMVMXR0.cpp 49903 2013-12-14 01:14:29Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu Pointer to the VMCPU.
1240	* @param uMsr The MSR.
1241	* @param uGuestMsr Value of the guest MSR.
1242	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1243	* necessary.
1244	*/
1245	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1246	{
1247	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1248	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1249	uint32_t i;
1250	for (i = 0; i < cMsrs; i++)
1251	{
1252	if (pGuestMsr->u32Msr == uMsr)
1253	break;
1254	pGuestMsr++;
1255	}
1256
1257	bool fAdded = false;
1258	if (i == cMsrs)
1259	{
1260	++cMsrs;
1261	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1262	AssertRCReturn(rc, rc);
1263
1264	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1265	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1266	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1267
1268	fAdded = true;
1269	}
1270
1271	/* Update the MSR values in the auto-load/store MSR area. */
1272	pGuestMsr->u32Msr = uMsr;
1273	pGuestMsr->u64Value = uGuestMsrValue;
1274
1275	/* Create/update the MSR slot in the host MSR area. */
1276	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1277	pHostMsr += i;
1278	pHostMsr->u32Msr = uMsr;
1279
1280	/*
1281	* Update the host MSR only when requested by the caller AND when we're
1282	* adding it to the auto-load/store area. Otherwise, it would have been
1283	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1284	*/
1285	if ( fAdded
1286	&& fUpdateHostMsr)
1287	{
1288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1290	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1291	}
1292
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1299	* auto-load/store MSR area in the VMCS.
1300	*
1301	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1302	* area.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu Pointer to the VMCPU.
1306	* @param uMsr The MSR.
1307	*/
1308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1309	{
1310	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1311	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1312	for (uint32_t i = 0; i < cMsrs; i++)
1313	{
1314	/* Find the MSR. */
1315	if (pGuestMsr->u32Msr == uMsr)
1316	{
1317	/* If it's the last MSR, simply reduce the count. */
1318	if (i == cMsrs - 1)
1319	{
1320	--cMsrs;
1321	break;
1322	}
1323
1324	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1325	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326	pLastGuestMsr += cMsrs;
1327	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1328	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1329
1330	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332	pLastHostMsr += cMsrs;
1333	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1334	pHostMsr->u64Value = pLastHostMsr->u64Value;
1335	--cMsrs;
1336	break;
1337	}
1338	pGuestMsr++;
1339	}
1340
1341	/* Update the VMCS if the count changed (meaning the MSR was found). */
1342	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1343	{
1344	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1345	AssertRCReturn(rc, rc);
1346
1347	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1349	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1350	}
1351
1352	return VINF_SUCCESS;
1353	}
1354
1355
1356	/**
1357	* Checks if the specified guest MSR is part of the auto-load/store area in
1358	* the VMCS.
1359	*
1360	* @returns true if found, false otherwise.
1361	* @param pVCpu Pointer to the VMCPU.
1362	* @param uMsr The MSR to find.
1363	*/
1364	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1365	{
1366	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368
1369	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1370	{
1371	if (pGuestMsr->u32Msr == uMsr)
1372	return true;
1373	}
1374	return false;
1375	}
1376
1377
1378	/**
1379	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1380	*
1381	* @param pVCpu Pointer to the VMCPU.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1386	{
1387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1388	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1393	{
1394	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1395	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1396	}
1397
1398	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1399	}
1400
1401
1402	#if HC_ARCH_BITS == 64
1403	/**
1404	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1405	* perform lazy restoration of the host MSRs while leaving VT-x.
1406	*
1407	* @param pVCpu Pointer to the VMCPU.
1408	*
1409	* @remarks No-long-jump zone!!!
1410	*/
1411	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1412	{
1413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1414
1415	/*
1416	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1417	*/
1418	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1419	{
1420	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1421	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1422	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1423	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1424	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1425	}
1426	}
1427
1428
1429	/**
1430	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1431	* lazily while leaving VT-x.
1432	*
1433	* @returns true if it does, false otherwise.
1434	* @param pVCpu Pointer to the VMCPU.
1435	* @param uMsr The MSR to check.
1436	*/
1437	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1438	{
1439	NOREF(pVCpu);
1440	switch (uMsr)
1441	{
1442	case MSR_K8_LSTAR:
1443	case MSR_K6_STAR:
1444	case MSR_K8_SF_MASK:
1445	case MSR_K8_KERNEL_GS_BASE:
1446	return true;
1447	}
1448	return false;
1449	}
1450
1451
1452	/**
1453	* Saves a set of guests MSRs back into the guest-CPU context.
1454	*
1455	* @param pVCpu Pointer to the VMCPU.
1456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1457	* out-of-sync. Make sure to update the required fields
1458	* before using them.
1459	*
1460	* @remarks No-long-jump zone!!!
1461	*/
1462	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1463	{
1464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1465	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1466
1467	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1468	{
1469	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1471	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1472	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1473	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1474	}
1475	}
1476
1477
1478	/**
1479	* Loads a set of guests MSRs to allow read/passthru to the guest.
1480	*
1481	* The name of this function is slightly confusing. This function does NOT
1482	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1483	* common prefix for functions dealing with "lazy restoration" of the shared
1484	* MSRs.
1485	*
1486	* @param pVCpu Pointer to the VMCPU.
1487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488	* out-of-sync. Make sure to update the required fields
1489	* before using them.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1499	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1500	{
1501	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1502	do { \
1503	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1504	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1505	else \
1506	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1507	} while (0)
1508
1509	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1510	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1511	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1513	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1514	}
1515	else
1516	{
1517	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1518	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1519	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1520	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1521	}
1522	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1523	}
1524
1525
1526	/**
1527	* Performs lazy restoration of the set of host MSRs if they were previously
1528	* loaded with guest MSR values.
1529	*
1530	* @param pVCpu Pointer to the VMCPU.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	* @remarks The guest MSRs should have been saved back into the guest-CPU
1534	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1535	*/
1536	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1537	{
1538	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1539	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1540
1541	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1542	{
1543	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1544	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1545	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1546	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1547	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1548	}
1549	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1550	}
1551	#endif /* HC_ARCH_BITS == 64 */
1552
1553
1554	#ifdef VBOX_STRICT
1555	/**
1556	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1557	* VMCS are correct.
1558	*
1559	* @param pVCpu Pointer to the VMCPU.
1560	*/
1561	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1562	{
1563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564
1565	/* Verify MSR counts in the VMCS are what we think it should be. */
1566	uint32_t cMsrs;
1567	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1568	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1569
1570	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1577	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1578	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1579	{
1580	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1581	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1582	pGuestMsr->u32Msr));
1583
1584	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1585	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1586	pHostMsr->u64Value, u64Msr));
1587
1588	/* Verify that the permissions are as expected in the MSR bitmap. */
1589	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1590	{
1591	VMXMSREXITREAD enmRead;
1592	VMXMSREXITWRITE enmWrite;
1593	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1594	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1595	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1596	pGuestMsr->u32Msr));
1597	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1598	pGuestMsr->u32Msr));
1599	}
1600	}
1601	}
1602	# endif /* VBOX_STRICT */
1603
1604
1605	/**
1606	* Flushes the TLB using EPT.
1607	*
1608	* @returns VBox status code.
1609	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1610	* enmFlush).
1611	* @param enmFlush Type of flush.
1612	*
1613	* @remarks Caller is responsible for making sure this function is called only
1614	* when NestedPaging is supported and providing @a enmFlush that is
1615	* supported by the CPU.
1616	* @remarks Can be called with interrupts disabled.
1617	*/
1618	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1619	{
1620	uint64_t au64Descriptor[2];
1621	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1622	au64Descriptor[0] = 0;
1623	else
1624	{
1625	Assert(pVCpu);
1626	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1627	}
1628	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1629
1630	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1631	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1632	rc));
1633	if ( RT_SUCCESS(rc)
1634	&& pVCpu)
1635	{
1636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1637	}
1638	}
1639
1640
1641	/**
1642	* Flushes the TLB using VPID.
1643	*
1644	* @returns VBox status code.
1645	* @param pVM Pointer to the VM.
1646	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1647	* enmFlush).
1648	* @param enmFlush Type of flush.
1649	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1650	* on @a enmFlush).
1651	*
1652	* @remarks Can be called with interrupts disabled.
1653	*/
1654	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1655	{
1656	NOREF(pVM);
1657	AssertPtr(pVM);
1658	Assert(pVM->hm.s.vmx.fVpid);
1659
1660	uint64_t au64Descriptor[2];
1661	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1662	{
1663	au64Descriptor[0] = 0;
1664	au64Descriptor[1] = 0;
1665	}
1666	else
1667	{
1668	AssertPtr(pVCpu);
1669	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1670	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1671	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1672	au64Descriptor[1] = GCPtr;
1673	}
1674
1675	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1676	AssertMsg(rc == VINF_SUCCESS,
1677	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1678	if ( RT_SUCCESS(rc)
1679	&& pVCpu)
1680	{
1681	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1682	}
1683	}
1684
1685
1686	/**
1687	* Invalidates a guest page by guest virtual address. Only relevant for
1688	* EPT/VPID, otherwise there is nothing really to invalidate.
1689	*
1690	* @returns VBox status code.
1691	* @param pVM Pointer to the VM.
1692	* @param pVCpu Pointer to the VMCPU.
1693	* @param GCVirt Guest virtual address of the page to invalidate.
1694	*/
1695	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1696	{
1697	AssertPtr(pVM);
1698	AssertPtr(pVCpu);
1699	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1700
1701	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1702	if (!fFlushPending)
1703	{
1704	/*
1705	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1706	* See @bugref{6043} and @bugref{6177}.
1707	*
1708	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1709	* function maybe called in a loop with individual addresses.
1710	*/
1711	if (pVM->hm.s.vmx.fVpid)
1712	{
1713	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1714	{
1715	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1716	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1717	}
1718	else
1719	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1720	}
1721	else if (pVM->hm.s.fNestedPaging)
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724
1725	return VINF_SUCCESS;
1726	}
1727
1728
1729	/**
1730	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1731	* otherwise there is nothing really to invalidate.
1732	*
1733	* @returns VBox status code.
1734	* @param pVM Pointer to the VM.
1735	* @param pVCpu Pointer to the VMCPU.
1736	* @param GCPhys Guest physical address of the page to invalidate.
1737	*/
1738	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1739	{
1740	NOREF(pVM); NOREF(GCPhys);
1741	LogFlowFunc(("%RGp\n", GCPhys));
1742
1743	/*
1744	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1745	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1746	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1747	*/
1748	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1749	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1750	return VINF_SUCCESS;
1751	}
1752
1753
1754	/**
1755	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1756	* case where neither EPT nor VPID is supported by the CPU.
1757	*
1758	* @param pVM Pointer to the VM.
1759	* @param pVCpu Pointer to the VMCPU.
1760	* @param pCpu Pointer to the global HM struct.
1761	*
1762	* @remarks Called with interrupts disabled.
1763	*/
1764	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1765	{
1766	AssertPtr(pVCpu);
1767	AssertPtr(pCpu);
1768	NOREF(pVM);
1769
1770	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1771
1772	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1773	#if 0
1774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1775	pVCpu->hm.s.TlbShootdown.cPages = 0;
1776	#endif
1777
1778	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1779	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1780	pVCpu->hm.s.fForceTLBFlush = false;
1781	return;
1782	}
1783
1784
1785	/**
1786	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1787	*
1788	* @param pVM Pointer to the VM.
1789	* @param pVCpu Pointer to the VMCPU.
1790	* @param pCpu Pointer to the global HM CPU struct.
1791	* @remarks All references to "ASID" in this function pertains to "VPID" in
1792	* Intel's nomenclature. The reason is, to avoid confusion in compare
1793	* statements since the host-CPU copies are named "ASID".
1794	*
1795	* @remarks Called with interrupts disabled.
1796	*/
1797	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1798	{
1799	#ifdef VBOX_WITH_STATISTICS
1800	bool fTlbFlushed = false;
1801	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1802	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1803	if (!fTlbFlushed) \
1804	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1805	} while (0)
1806	#else
1807	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1808	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1809	#endif
1810
1811	AssertPtr(pVM);
1812	AssertPtr(pCpu);
1813	AssertPtr(pVCpu);
1814	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1815	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1816	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1817
1818
1819	/*
1820	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1821	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1822	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1823	*/
1824	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1825	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1826	{
1827	++pCpu->uCurrentAsid;
1828	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1829	{
1830	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1831	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1832	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1833	}
1834
1835	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1836	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1837	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1838
1839	/*
1840	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1841	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1842	*/
1843	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1844	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1845	HMVMX_SET_TAGGED_TLB_FLUSHED();
1846	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1847	}
1848
1849	/* Check for explicit TLB shootdowns. */
1850	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1851	{
1852	/*
1853	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1854	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1855	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1856	* but not guest-physical mappings.
1857	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1858	*/
1859	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1860	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1861	HMVMX_SET_TAGGED_TLB_FLUSHED();
1862	}
1863
1864	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1865	* where it is commented out. Support individual entry flushing
1866	* someday. */
1867	#if 0
1868	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1869	{
1870	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1871
1872	/*
1873	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1874	* as supported by the CPU.
1875	*/
1876	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1877	{
1878	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1879	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1880	}
1881	else
1882	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1883
1884	HMVMX_SET_TAGGED_TLB_FLUSHED();
1885	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1886	pVCpu->hm.s.TlbShootdown.cPages = 0;
1887	}
1888	#endif
1889
1890	pVCpu->hm.s.fForceTLBFlush = false;
1891
1892	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1893
1894	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1895	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1896	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1897	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1898	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1899	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1900	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1901	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1902
1903	/* Update VMCS with the VPID. */
1904	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1905	AssertRC(rc);
1906
1907	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1908	}
1909
1910
1911	/**
1912	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	* @param pVCpu Pointer to the VMCPU.
1917	* @param pCpu Pointer to the global HM CPU struct.
1918	*
1919	* @remarks Called with interrupts disabled.
1920	*/
1921	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922	{
1923	AssertPtr(pVM);
1924	AssertPtr(pVCpu);
1925	AssertPtr(pCpu);
1926	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1927	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1928
1929	/*
1930	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1931	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1932	*/
1933	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1934	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1935	{
1936	pVCpu->hm.s.fForceTLBFlush = true;
1937	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1938	}
1939
1940	/* Check for explicit TLB shootdown flushes. */
1941	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1942	{
1943	pVCpu->hm.s.fForceTLBFlush = true;
1944	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1945	}
1946
1947	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1948	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1949
1950	if (pVCpu->hm.s.fForceTLBFlush)
1951	{
1952	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1953	pVCpu->hm.s.fForceTLBFlush = false;
1954	}
1955	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1956	* where it is commented out. Support individual entry flushing
1957	* someday. */
1958	#if 0
1959	else
1960	{
1961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1962	{
1963	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1964	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1966	}
1967	else
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1969
1970	pVCpu->hm.s.TlbShootdown.cPages = 0;
1971	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972	}
1973	#endif
1974	}
1975
1976
1977	/**
1978	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1979	*
1980	* @returns VBox status code.
1981	* @param pVM Pointer to the VM.
1982	* @param pVCpu Pointer to the VMCPU.
1983	* @param pCpu Pointer to the global HM CPU struct.
1984	*
1985	* @remarks Called with interrupts disabled.
1986	*/
1987	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1988	{
1989	AssertPtr(pVM);
1990	AssertPtr(pVCpu);
1991	AssertPtr(pCpu);
1992	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1993	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1994
1995	/*
1996	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1997	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1998	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1999	*/
2000	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2001	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2002	{
2003	pVCpu->hm.s.fForceTLBFlush = true;
2004	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2005	}
2006
2007	/* Check for explicit TLB shootdown flushes. */
2008	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2009	{
2010	/*
2011	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2012	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2013	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2014	*/
2015	pVCpu->hm.s.fForceTLBFlush = true;
2016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2017	}
2018
2019	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2020	if (pVCpu->hm.s.fForceTLBFlush)
2021	{
2022	++pCpu->uCurrentAsid;
2023	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2024	{
2025	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2026	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2027	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2032	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2033	if (pCpu->fFlushAsidBeforeUse)
2034	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2035	}
2036	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2037	* where it is commented out. Support individual entry flushing
2038	* someday. */
2039	#if 0
2040	else
2041	{
2042	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2043	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2044	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2045	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2046
2047	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2048	{
2049	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2050	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2051	{
2052	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2053	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2054	}
2055	else
2056	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2057
2058	pVCpu->hm.s.TlbShootdown.cPages = 0;
2059	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060	}
2061	else
2062	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2063	}
2064	#endif
2065
2066	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2067	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2068	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2069	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2070	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2074	AssertRC(rc);
2075	}
2076
2077
2078	/**
2079	* Flushes the guest TLB entry based on CPU capabilities.
2080	*
2081	* @param pVCpu Pointer to the VMCPU.
2082	* @param pCpu Pointer to the global HM CPU struct.
2083	*/
2084	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2087	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2088	#endif
2089	PVM pVM = pVCpu->CTX_SUFF(pVM);
2090	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2091	{
2092	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2093	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2094	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2095	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2096	default:
2097	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2098	break;
2099	}
2100
2101	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2102	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2103
2104	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2105	}
2106
2107
2108	/**
2109	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2110	* TLB entries from the host TLB before VM-entry.
2111	*
2112	* @returns VBox status code.
2113	* @param pVM Pointer to the VM.
2114	*/
2115	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2116	{
2117	/*
2118	* Determine optimal flush type for Nested Paging.
2119	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2120	* guest execution (see hmR3InitFinalizeR0()).
2121	*/
2122	if (pVM->hm.s.fNestedPaging)
2123	{
2124	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2125	{
2126	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2127	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2128	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2129	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2130	else
2131	{
2132	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2133	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2134	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2135	}
2136
2137	/* Make sure the write-back cacheable memory type for EPT is supported. */
2138	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2139	{
2140	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144	}
2145	else
2146	{
2147	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2148	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2149	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2150	}
2151	}
2152
2153	/*
2154	* Determine optimal flush type for VPID.
2155	*/
2156	if (pVM->hm.s.vmx.fVpid)
2157	{
2158	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2159	{
2160	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2161	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2162	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2164	else
2165	{
2166	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2167	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2169	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2170	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2172	pVM->hm.s.vmx.fVpid = false;
2173	}
2174	}
2175	else
2176	{
2177	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2178	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183
2184	/*
2185	* Setup the handler for flushing tagged-TLBs.
2186	*/
2187	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2188	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2189	else if (pVM->hm.s.fNestedPaging)
2190	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2191	else if (pVM->hm.s.vmx.fVpid)
2192	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2193	else
2194	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2195	return VINF_SUCCESS;
2196	}
2197
2198
2199	/**
2200	* Sets up pin-based VM-execution controls in the VMCS.
2201	*
2202	* @returns VBox status code.
2203	* @param pVM Pointer to the VM.
2204	* @param pVCpu Pointer to the VMCPU.
2205	*/
2206	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2207	{
2208	AssertPtr(pVM);
2209	AssertPtr(pVCpu);
2210
2211	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2212	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2213
2214	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2215	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2216	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2217
2218	/* Enable the VMX preemption timer. */
2219	if (pVM->hm.s.vmx.fUsePreemptTimer)
2220	{
2221	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2223	}
2224
2225	if ((val & zap) != val)
2226	{
2227	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2228	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2229	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2230	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2231	}
2232
2233	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2234	AssertRCReturn(rc, rc);
2235
2236	/* Update VCPU with the currently set pin-based VM-execution controls. */
2237	pVCpu->hm.s.vmx.u32PinCtls = val;
2238	return rc;
2239	}
2240
2241
2242	/**
2243	* Sets up processor-based VM-execution controls in the VMCS.
2244	*
2245	* @returns VBox status code.
2246	* @param pVM Pointer to the VM.
2247	* @param pVMCPU Pointer to the VMCPU.
2248	*/
2249	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2250	{
2251	AssertPtr(pVM);
2252	AssertPtr(pVCpu);
2253
2254	int rc = VERR_INTERNAL_ERROR_5;
2255	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2256	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2257
2258	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2259	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2260	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2261	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2265
2266	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2267	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2268	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2269	{
2270	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2271	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2272	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273	}
2274
2275	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2276	if (!pVM->hm.s.fNestedPaging)
2277	{
2278	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2279	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2280	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2281	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2282	}
2283
2284	/* Use TPR shadowing if supported by the CPU. */
2285	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2286	{
2287	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2288	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2289	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2290	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	AssertRCReturn(rc, rc);
2292
2293	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2294	/* CR8 writes causes a VM-exit based on TPR threshold. */
2295	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2296	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2297	}
2298	else
2299	{
2300	/*
2301	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2302	* Set this control only for 64-bit guests.
2303	*/
2304	if (pVM->hm.s.fAllow64BitGuests)
2305	{
2306	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2307	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2308	}
2309	}
2310
2311	/* Use MSR-bitmaps if supported by the CPU. */
2312	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2315
2316	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2317	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2318	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2319	AssertRCReturn(rc, rc);
2320
2321	/*
2322	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2323	* automatically as dedicated fields in the VMCS.
2324	*/
2325	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2326	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2327	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330
2331	#if HC_ARCH_BITS == 64
2332	/*
2333	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2334	*/
2335	if (pVM->hm.s.fAllow64BitGuests)
2336	{
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	}
2342	#endif
2343	}
2344
2345	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2346	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2347	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2348
2349	if ((val & zap) != val)
2350	{
2351	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2352	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2353	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2355	}
2356
2357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2358	AssertRCReturn(rc, rc);
2359
2360	/* Update VCPU with the currently set processor-based VM-execution controls. */
2361	pVCpu->hm.s.vmx.u32ProcCtls = val;
2362
2363	/*
2364	* Secondary processor-based VM-execution controls.
2365	*/
2366	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2367	{
2368	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2373
2374	if (pVM->hm.s.fNestedPaging)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2376	else
2377	{
2378	/*
2379	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2380	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2381	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2382	*/
2383	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2385	}
2386
2387	if (pVM->hm.s.vmx.fVpid)
2388	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2389
2390	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2392
2393	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2394	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2395	* done dynamically. */
2396	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2397	{
2398	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2399	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2401	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2402	AssertRCReturn(rc, rc);
2403	}
2404
2405	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2406	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2407
2408	if ((val & zap) != val)
2409	{
2410	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2411	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2412	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2413	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2414	}
2415
2416	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2417	AssertRCReturn(rc, rc);
2418
2419	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2420	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2421	}
2422	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2423	{
2424	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2425	"available\n"));
2426	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2427	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2428	}
2429
2430	return VINF_SUCCESS;
2431	}
2432
2433
2434	/**
2435	* Sets up miscellaneous (everything other than Pin & Processor-based
2436	* VM-execution) control fields in the VMCS.
2437	*
2438	* @returns VBox status code.
2439	* @param pVM Pointer to the VM.
2440	* @param pVCpu Pointer to the VMCPU.
2441	*/
2442	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2443	{
2444	NOREF(pVM);
2445	AssertPtr(pVM);
2446	AssertPtr(pVCpu);
2447
2448	int rc = VERR_GENERAL_FAILURE;
2449
2450	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2451	#if 0
2452	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2453	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2455
2456	/*
2457	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2458	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2459	* We thus use the exception bitmap to control it rather than use both.
2460	*/
2461	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2463
2464	/** @todo Explore possibility of using IO-bitmaps. */
2465	/* All IO & IOIO instructions cause VM-exits. */
2466	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2467	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2468
2469	/* Initialize the MSR-bitmap area. */
2470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2473	#endif
2474
2475	/* Setup MSR auto-load/store area. */
2476	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2477	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2478	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2479	AssertRCReturn(rc, rc);
2480	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2481	AssertRCReturn(rc, rc);
2482
2483	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2484	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2485	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2486	AssertRCReturn(rc, rc);
2487
2488	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2489	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2490	AssertRCReturn(rc, rc);
2491
2492	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2493	#if 0
2494	/* Setup debug controls */
2495	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2496	AssertRCReturn(rc, rc);
2497	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2498	AssertRCReturn(rc, rc);
2499	#endif
2500
2501	return rc;
2502	}
2503
2504
2505	/**
2506	* Sets up the initial exception bitmap in the VMCS based on static conditions
2507	* (i.e. conditions that cannot ever change after starting the VM).
2508	*
2509	* @returns VBox status code.
2510	* @param pVM Pointer to the VM.
2511	* @param pVCpu Pointer to the VMCPU.
2512	*/
2513	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2514	{
2515	AssertPtr(pVM);
2516	AssertPtr(pVCpu);
2517
2518	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2519
2520	uint32_t u32XcptBitmap = 0;
2521
2522	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2523	if (!pVM->hm.s.fNestedPaging)
2524	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2525
2526	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2527	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2528	AssertRCReturn(rc, rc);
2529	return rc;
2530	}
2531
2532
2533	/**
2534	* Sets up the initial guest-state mask. The guest-state mask is consulted
2535	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2536	* for the nested virtualization case (as it would cause a VM-exit).
2537	*
2538	* @param pVCpu Pointer to the VMCPU.
2539	*/
2540	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2541	{
2542	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2543	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2544	return VINF_SUCCESS;
2545	}
2546
2547
2548	/**
2549	* Does per-VM VT-x initialization.
2550	*
2551	* @returns VBox status code.
2552	* @param pVM Pointer to the VM.
2553	*/
2554	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2555	{
2556	LogFlowFunc(("pVM=%p\n", pVM));
2557
2558	int rc = hmR0VmxStructsAlloc(pVM);
2559	if (RT_FAILURE(rc))
2560	{
2561	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2562	return rc;
2563	}
2564
2565	return VINF_SUCCESS;
2566	}
2567
2568
2569	/**
2570	* Does per-VM VT-x termination.
2571	*
2572	* @returns VBox status code.
2573	* @param pVM Pointer to the VM.
2574	*/
2575	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2576	{
2577	LogFlowFunc(("pVM=%p\n", pVM));
2578
2579	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2580	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2581	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2582	#endif
2583	hmR0VmxStructsFree(pVM);
2584	return VINF_SUCCESS;
2585	}
2586
2587
2588	/**
2589	* Sets up the VM for execution under VT-x.
2590	* This function is only called once per-VM during initialization.
2591	*
2592	* @returns VBox status code.
2593	* @param pVM Pointer to the VM.
2594	*/
2595	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2596	{
2597	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2598	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2599
2600	LogFlowFunc(("pVM=%p\n", pVM));
2601
2602	/*
2603	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2604	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2605	*/
2606	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2607	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2608	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2609	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2610	{
2611	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2612	return VERR_INTERNAL_ERROR;
2613	}
2614
2615	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2616	/*
2617	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2618	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2619	*/
2620	if ( pVM->hm.s.fAllow64BitGuests
2621	&& !HMVMX_IS_64BIT_HOST_MODE())
2622	{
2623	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2624	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2625	}
2626	#endif
2627
2628	/* Initialize these always, see hmR3InitFinalizeR0().*/
2629	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2630	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2631
2632	/* Setup the tagged-TLB flush handlers. */
2633	int rc = hmR0VmxSetupTaggedTlb(pVM);
2634	if (RT_FAILURE(rc))
2635	{
2636	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2637	return rc;
2638	}
2639
2640	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2641	{
2642	PVMCPU pVCpu = &pVM->aCpus[i];
2643	AssertPtr(pVCpu);
2644	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2645
2646	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2647	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2648
2649	/* Set revision dword at the beginning of the VMCS structure. */
2650	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2651
2652	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2653	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2654	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2655	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2656
2657	/* Load this VMCS as the current VMCS. */
2658	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2659	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2660	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2661
2662	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2663	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2664	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2665
2666	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2667	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2668	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2669
2670	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2671	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2672	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2673
2674	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2675	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2676	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2677
2678	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2679	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2680	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2681
2682	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2683	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2684	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686	#endif
2687
2688	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2689	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2690	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2691	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2692
2693	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2694
2695	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2696	}
2697
2698	return VINF_SUCCESS;
2699	}
2700
2701
2702	/**
2703	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2704	* the VMCS.
2705	*
2706	* @returns VBox status code.
2707	* @param pVM Pointer to the VM.
2708	* @param pVCpu Pointer to the VMCPU.
2709	*/
2710	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2711	{
2712	NOREF(pVM); NOREF(pVCpu);
2713
2714	RTCCUINTREG uReg = ASMGetCR0();
2715	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2716	AssertRCReturn(rc, rc);
2717
2718	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2719	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2720	if (HMVMX_IS_64BIT_HOST_MODE())
2721	{
2722	uint64_t uRegCR3 = HMR0Get64bitCR3();
2723	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2724	}
2725	else
2726	#endif
2727	{
2728	uReg = ASMGetCR3();
2729	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2730	}
2731	AssertRCReturn(rc, rc);
2732
2733	uReg = ASMGetCR4();
2734	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2735	AssertRCReturn(rc, rc);
2736	return rc;
2737	}
2738
2739
2740	#if HC_ARCH_BITS == 64
2741	/**
2742	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2743	* requirements. See hmR0VmxSaveHostSegmentRegs().
2744	*/
2745	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2746	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2747	{ \
2748	bool fValidSelector = true; \
2749	if ((selValue) & X86_SEL_LDT) \
2750	{ \
2751	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2752	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2753	} \
2754	if (fValidSelector) \
2755	{ \
2756	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2757	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2758	} \
2759	(selValue) = 0; \
2760	}
2761	#endif
2762
2763
2764	/**
2765	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2766	* the host-state area in the VMCS.
2767	*
2768	* @returns VBox status code.
2769	* @param pVM Pointer to the VM.
2770	* @param pVCpu Pointer to the VMCPU.
2771	*/
2772	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2773	{
2774	NOREF(pVM);
2775	int rc = VERR_INTERNAL_ERROR_5;
2776
2777	/*
2778	* Host DS, ES, FS and GS segment registers.
2779	*/
2780	#if HC_ARCH_BITS == 64
2781	RTSEL uSelDS = ASMGetDS();
2782	RTSEL uSelES = ASMGetES();
2783	RTSEL uSelFS = ASMGetFS();
2784	RTSEL uSelGS = ASMGetGS();
2785	#else
2786	RTSEL uSelDS = 0;
2787	RTSEL uSelES = 0;
2788	RTSEL uSelFS = 0;
2789	RTSEL uSelGS = 0;
2790	#endif
2791
2792	/* Recalculate which host-state bits need to be manually restored. */
2793	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2794
2795	/*
2796	* Host CS and SS segment registers.
2797	*/
2798	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2799	RTSEL uSelCS;
2800	RTSEL uSelSS;
2801	if (HMVMX_IS_64BIT_HOST_MODE())
2802	{
2803	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2804	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2805	}
2806	else
2807	{
2808	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2809	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2810	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2811	}
2812	#else
2813	RTSEL uSelCS = ASMGetCS();
2814	RTSEL uSelSS = ASMGetSS();
2815	#endif
2816
2817	/*
2818	* Host TR segment register.
2819	*/
2820	RTSEL uSelTR = ASMGetTR();
2821
2822	#if HC_ARCH_BITS == 64
2823	/*
2824	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2825	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2826	*/
2827	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2828	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2829	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2830	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2831	# undef VMXLOCAL_ADJUST_HOST_SEG
2832	#endif
2833
2834	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2835	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2836	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2837	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2838	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2839	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2840	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2841	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2842	Assert(uSelCS);
2843	Assert(uSelTR);
2844
2845	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2846	#if 0
2847	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2848	Assert(uSelSS != 0);
2849	#endif
2850
2851	/* Write these host selector fields into the host-state area in the VMCS. */
2852	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2853	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2854	#if HC_ARCH_BITS == 64
2855	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2856	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2857	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2858	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2859	#endif
2860	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2861
2862	/*
2863	* Host GDTR and IDTR.
2864	*/
2865	RTGDTR Gdtr;
2866	RT_ZERO(Gdtr);
2867	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2868	if (HMVMX_IS_64BIT_HOST_MODE())
2869	{
2870	X86XDTR64 Gdtr64;
2871	X86XDTR64 Idtr64;
2872	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2873	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2874	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2875
2876	Gdtr.cbGdt = Gdtr64.cb;
2877	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2878	}
2879	else
2880	#endif
2881	{
2882	RTIDTR Idtr;
2883	ASMGetGDTR(&Gdtr);
2884	ASMGetIDTR(&Idtr);
2885	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2886	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2887
2888	#if HC_ARCH_BITS == 64
2889	/*
2890	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2891	* maximum limit (0xffff) on every VM-exit.
2892	*/
2893	if (Gdtr.cbGdt != 0xffff)
2894	{
2895	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2896	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2897	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2898	}
2899
2900	/*
2901	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2902	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2903	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2904	*/
2905	if (Idtr.cbIdt < 0x0fff)
2906	{
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2908	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2909	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2910	}
2911	#endif
2912	}
2913
2914	/*
2915	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2916	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2917	*/
2918	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2919	{
2920	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2921	return VERR_VMX_INVALID_HOST_STATE;
2922	}
2923
2924	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2925	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2926	if (HMVMX_IS_64BIT_HOST_MODE())
2927	{
2928	/* We need the 64-bit TR base for hybrid darwin. */
2929	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2930	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2931	}
2932	else
2933	#endif
2934	{
2935	uintptr_t uTRBase;
2936	#if HC_ARCH_BITS == 64
2937	uTRBase = X86DESC64_BASE(pDesc);
2938
2939	/*
2940	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2941	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2942	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2943	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2944	*
2945	* [1] See Intel spec. 3.5 "System Descriptor Types".
2946	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2947	*/
2948	Assert(pDesc->System.u4Type == 11);
2949	if ( pDesc->System.u16LimitLow != 0x67
2950	\|\| pDesc->System.u4LimitHigh)
2951	{
2952	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2953	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2954
2955	/* Store the GDTR here as we need it while restoring TR. */
2956	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2957	}
2958	#else
2959	uTRBase = X86DESC_BASE(pDesc);
2960	#endif
2961	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2962	}
2963	AssertRCReturn(rc, rc);
2964
2965	/*
2966	* Host FS base and GS base.
2967	*/
2968	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2969	if (HMVMX_IS_64BIT_HOST_MODE())
2970	{
2971	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2972	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2973	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2974	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2975
2976	# if HC_ARCH_BITS == 64
2977	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2978	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2979	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2980	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2981	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2982	# endif
2983	}
2984	#endif
2985	return rc;
2986	}
2987
2988
2989	/**
2990	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2991	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2992	* the host after every successful VM-exit.
2993	*
2994	* @returns VBox status code.
2995	* @param pVM Pointer to the VM.
2996	* @param pVCpu Pointer to the VMCPU.
2997	*
2998	* @remarks No-long-jump zone!!!
2999	*/
3000	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3001	{
3002	NOREF(pVM);
3003
3004	AssertPtr(pVCpu);
3005	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3006
3007	int rc = VINF_SUCCESS;
3008	#if HC_ARCH_BITS == 64
3009	if (pVM->hm.s.fAllow64BitGuests)
3010	hmR0VmxLazySaveHostMsrs(pVCpu);
3011	#endif
3012
3013	if (pVCpu->hm.s.vmx.cMsrs > 0)
3014	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3015
3016	/*
3017	* Host Sysenter MSRs.
3018	*/
3019	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3020	AssertRCReturn(rc, rc);
3021	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3022	if (HMVMX_IS_64BIT_HOST_MODE())
3023	{
3024	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3025	AssertRCReturn(rc, rc);
3026	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3027	}
3028	else
3029	{
3030	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3031	AssertRCReturn(rc, rc);
3032	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3033	}
3034	#elif HC_ARCH_BITS == 32
3035	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3036	AssertRCReturn(rc, rc);
3037	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3038	#else
3039	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3040	AssertRCReturn(rc, rc);
3041	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3042	#endif
3043	AssertRCReturn(rc, rc);
3044
3045	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3046	* hmR0VmxSetupExitCtls() !! */
3047	return rc;
3048	}
3049
3050
3051	/**
3052	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3053	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3054	* controls".
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu Pointer to the VMCPU.
3058	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3059	* out-of-sync. Make sure to update the required fields
3060	* before using them.
3061	*
3062	* @remarks No-long-jump zone!!!
3063	*/
3064	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3065	{
3066	int rc = VINF_SUCCESS;
3067	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3068	{
3069	PVM pVM = pVCpu->CTX_SUFF(pVM);
3070	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3071	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3072
3073	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3074	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3075
3076	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3077	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3078	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3079	else
3080	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3081
3082	/*
3083	* The following should -not- be set (since we're not in SMM mode):
3084	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3085	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3086	*/
3087
3088	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3089	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3090	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3091
3092	if ((val & zap) != val)
3093	{
3094	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3095	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3096	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3097	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3098	}
3099
3100	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3101	AssertRCReturn(rc, rc);
3102
3103	/* Update VCPU with the currently set VM-exit controls. */
3104	pVCpu->hm.s.vmx.u32EntryCtls = val;
3105	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3106	}
3107	return rc;
3108	}
3109
3110
3111	/**
3112	* Sets up the VM-exit controls in the VMCS.
3113	*
3114	* @returns VBox status code.
3115	* @param pVM Pointer to the VM.
3116	* @param pVCpu Pointer to the VMCPU.
3117	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3118	* out-of-sync. Make sure to update the required fields
3119	* before using them.
3120	*
3121	* @remarks requires EFER.
3122	*/
3123	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3124	{
3125	NOREF(pMixedCtx);
3126
3127	int rc = VINF_SUCCESS;
3128	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3129	{
3130	PVM pVM = pVCpu->CTX_SUFF(pVM);
3131	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3132	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3133
3134	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3135	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3136
3137	/*
3138	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3139	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3140	*/
3141	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3142	if (HMVMX_IS_64BIT_HOST_MODE())
3143	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3144	else
3145	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3146	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3147	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3148	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3149	else
3150	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3151	#endif
3152
3153	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3154	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3155
3156	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3157	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3158	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3159	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3160	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3161
3162	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3163	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3164
3165	if ((val & zap) != val)
3166	{
3167	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3168	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3169	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3170	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3171	}
3172
3173	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3174	AssertRCReturn(rc, rc);
3175
3176	/* Update VCPU with the currently set VM-exit controls. */
3177	pVCpu->hm.s.vmx.u32ExitCtls = val;
3178	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3179	}
3180	return rc;
3181	}
3182
3183
3184	/**
3185	* Loads the guest APIC and related state.
3186	*
3187	* @returns VBox status code.
3188	* @param pVM Pointer to the VM.
3189	* @param pVCpu Pointer to the VMCPU.
3190	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3191	* out-of-sync. Make sure to update the required fields
3192	* before using them.
3193	*/
3194	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3195	{
3196	NOREF(pMixedCtx);
3197
3198	int rc = VINF_SUCCESS;
3199	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3200	{
3201	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3202	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3203	{
3204	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3205
3206	bool fPendingIntr = false;
3207	uint8_t u8Tpr = 0;
3208	uint8_t u8PendingIntr = 0;
3209	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3210	AssertRCReturn(rc, rc);
3211
3212	/*
3213	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3214	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3215	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3216	* the interrupt when we VM-exit for other reasons.
3217	*/
3218	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3219	uint32_t u32TprThreshold = 0;
3220	if (fPendingIntr)
3221	{
3222	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3223	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3224	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3225	if (u8PendingPriority <= u8TprPriority)
3226	u32TprThreshold = u8PendingPriority;
3227	else
3228	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3229	}
3230	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3231
3232	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3233	AssertRCReturn(rc, rc);
3234	}
3235
3236	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3237	}
3238	return rc;
3239	}
3240
3241
3242	/**
3243	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3244	*
3245	* @returns Guest's interruptibility-state.
3246	* @param pVCpu Pointer to the VMCPU.
3247	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3248	* out-of-sync. Make sure to update the required fields
3249	* before using them.
3250	*
3251	* @remarks No-long-jump zone!!!
3252	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3253	*/
3254	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3255	{
3256	/*
3257	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3258	* inhibit interrupts or clear any existing interrupt-inhibition.
3259	*/
3260	uint32_t uIntrState = 0;
3261	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3262	{
3263	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3264	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3265	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3266	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3267	{
3268	/*
3269	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3270	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3271	*/
3272	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3273	}
3274	else if (pMixedCtx->eflags.Bits.u1IF)
3275	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3276	else
3277	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3278	}
3279	return uIntrState;
3280	}
3281
3282
3283	/**
3284	* Loads the guest's interruptibility-state into the guest-state area in the
3285	* VMCS.
3286	*
3287	* @returns VBox status code.
3288	* @param pVCpu Pointer to the VMCPU.
3289	* @param uIntrState The interruptibility-state to set.
3290	*/
3291	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3292	{
3293	NOREF(pVCpu);
3294	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3295	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3296	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3297	AssertRCReturn(rc, rc);
3298	return rc;
3299	}
3300
3301
3302	/**
3303	* Loads the guest's RIP into the guest-state area in the VMCS.
3304	*
3305	* @returns VBox status code.
3306	* @param pVCpu Pointer to the VMCPU.
3307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3308	* out-of-sync. Make sure to update the required fields
3309	* before using them.
3310	*
3311	* @remarks No-long-jump zone!!!
3312	*/
3313	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3314	{
3315	int rc = VINF_SUCCESS;
3316	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3317	{
3318	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3319	AssertRCReturn(rc, rc);
3320
3321	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3322	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3323	}
3324	return rc;
3325	}
3326
3327
3328	/**
3329	* Loads the guest's RSP into the guest-state area in the VMCS.
3330	*
3331	* @returns VBox status code.
3332	* @param pVCpu Pointer to the VMCPU.
3333	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3334	* out-of-sync. Make sure to update the required fields
3335	* before using them.
3336	*
3337	* @remarks No-long-jump zone!!!
3338	*/
3339	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3340	{
3341	int rc = VINF_SUCCESS;
3342	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3343	{
3344	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3345	AssertRCReturn(rc, rc);
3346
3347	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3348	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3349	}
3350	return rc;
3351	}
3352
3353
3354	/**
3355	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3356	*
3357	* @returns VBox status code.
3358	* @param pVCpu Pointer to the VMCPU.
3359	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3360	* out-of-sync. Make sure to update the required fields
3361	* before using them.
3362	*
3363	* @remarks No-long-jump zone!!!
3364	*/
3365	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3366	{
3367	int rc = VINF_SUCCESS;
3368	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3369	{
3370	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3371	Let us assert it as such and use 32-bit VMWRITE. */
3372	Assert(!(pMixedCtx->rflags.u64 >> 32));
3373	X86EFLAGS Eflags = pMixedCtx->eflags;
3374	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3375	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3376
3377	/*
3378	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3379	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3380	*/
3381	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3382	{
3383	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3384	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3385	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3386	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3387	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3388	}
3389
3390	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3391	AssertRCReturn(rc, rc);
3392
3393	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3394	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3395	}
3396	return rc;
3397	}
3398
3399
3400	/**
3401	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3402	*
3403	* @returns VBox status code.
3404	* @param pVCpu Pointer to the VMCPU.
3405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3406	* out-of-sync. Make sure to update the required fields
3407	* before using them.
3408	*
3409	* @remarks No-long-jump zone!!!
3410	*/
3411	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3412	{
3413	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3414	AssertRCReturn(rc, rc);
3415	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3416	AssertRCReturn(rc, rc);
3417	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3418	AssertRCReturn(rc, rc);
3419	return rc;
3420	}
3421
3422
3423	/**
3424	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3425	* CR0 is partially shared with the host and we have to consider the FPU bits.
3426	*
3427	* @returns VBox status code.
3428	* @param pVM Pointer to the VM.
3429	* @param pVCpu Pointer to the VMCPU.
3430	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3431	* out-of-sync. Make sure to update the required fields
3432	* before using them.
3433	*
3434	* @remarks No-long-jump zone!!!
3435	*/
3436	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3437	{
3438	/*
3439	* Guest CR0.
3440	* Guest FPU.
3441	*/
3442	int rc = VINF_SUCCESS;
3443	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3444	{
3445	Assert(!(pMixedCtx->cr0 >> 32));
3446	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3447	PVM pVM = pVCpu->CTX_SUFF(pVM);
3448
3449	/* The guest's view (read access) of its CR0 is unblemished. */
3450	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3451	AssertRCReturn(rc, rc);
3452	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3453
3454	/* Setup VT-x's view of the guest CR0. */
3455	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3456	if (pVM->hm.s.fNestedPaging)
3457	{
3458	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3459	{
3460	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3461	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3462	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3463	}
3464	else
3465	{
3466	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3467	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3468	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3469	}
3470
3471	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3472	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3473	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3474
3475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3476	AssertRCReturn(rc, rc);
3477	}
3478	else
3479	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3480
3481	/*
3482	* Guest FPU bits.
3483	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3484	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3485	*/
3486	u32GuestCR0 \|= X86_CR0_NE;
3487	bool fInterceptNM = false;
3488	if (CPUMIsGuestFPUStateActive(pVCpu))
3489	{
3490	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3491	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3492	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3493	}
3494	else
3495	{
3496	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3497	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3498	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3499	}
3500
3501	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3502	bool fInterceptMF = false;
3503	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3504	fInterceptMF = true;
3505
3506	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3507	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3508	{
3509	Assert(PDMVmmDevHeapIsEnabled(pVM));
3510	Assert(pVM->hm.s.vmx.pRealModeTSS);
3511	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3512	fInterceptNM = true;
3513	fInterceptMF = true;
3514	}
3515	else
3516	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3517
3518	if (fInterceptNM)
3519	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3520	else
3521	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3522
3523	if (fInterceptMF)
3524	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3525	else
3526	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3527
3528	/* Additional intercepts for debugging, define these yourself explicitly. */
3529	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3531	\| RT_BIT(X86_XCPT_BP)
3532	\| RT_BIT(X86_XCPT_DB)
3533	\| RT_BIT(X86_XCPT_DE)
3534	\| RT_BIT(X86_XCPT_NM)
3535	\| RT_BIT(X86_XCPT_UD)
3536	\| RT_BIT(X86_XCPT_NP)
3537	\| RT_BIT(X86_XCPT_SS)
3538	\| RT_BIT(X86_XCPT_GP)
3539	\| RT_BIT(X86_XCPT_PF)
3540	\| RT_BIT(X86_XCPT_MF)
3541	;
3542	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3543	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3544	#endif
3545
3546	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3547
3548	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3549	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3550	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3551	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3552	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3553	else
3554	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3555
3556	u32GuestCR0 \|= uSetCR0;
3557	u32GuestCR0 &= uZapCR0;
3558	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3559
3560	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3561	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3562	AssertRCReturn(rc, rc);
3563	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3564	AssertRCReturn(rc, rc);
3565	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3566
3567	/*
3568	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3569	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3570	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3571	*/
3572	uint32_t u32CR0Mask = 0;
3573	u32CR0Mask = X86_CR0_PE
3574	\| X86_CR0_NE
3575	\| X86_CR0_WP
3576	\| X86_CR0_PG
3577	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3578	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3579	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3580
3581	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3582	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3583	* and @bugref{6944}. */
3584	#if 0
3585	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3586	u32CR0Mask &= ~X86_CR0_PE;
3587	#endif
3588	if (pVM->hm.s.fNestedPaging)
3589	u32CR0Mask &= ~X86_CR0_WP;
3590
3591	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3592	if (fInterceptNM)
3593	{
3594	u32CR0Mask \|= X86_CR0_TS
3595	\| X86_CR0_MP;
3596	}
3597
3598	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3599	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3600	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3601	AssertRCReturn(rc, rc);
3602	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3603
3604	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3605	}
3606	return rc;
3607	}
3608
3609
3610	/**
3611	* Loads the guest control registers (CR3, CR4) into the guest-state area
3612	* in the VMCS.
3613	*
3614	* @returns VBox status code.
3615	* @param pVM Pointer to the VM.
3616	* @param pVCpu Pointer to the VMCPU.
3617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3618	* out-of-sync. Make sure to update the required fields
3619	* before using them.
3620	*
3621	* @remarks No-long-jump zone!!!
3622	*/
3623	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3624	{
3625	int rc = VINF_SUCCESS;
3626	PVM pVM = pVCpu->CTX_SUFF(pVM);
3627
3628	/*
3629	* Guest CR2.
3630	* It's always loaded in the assembler code. Nothing to do here.
3631	*/
3632
3633	/*
3634	* Guest CR3.
3635	*/
3636	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3637	{
3638	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3639	if (pVM->hm.s.fNestedPaging)
3640	{
3641	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3642
3643	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3644	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3645	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3646	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3647
3648	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3649	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3650	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3651
3652	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3653	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3654	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3655	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3656
3657	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3658	AssertRCReturn(rc, rc);
3659	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3660
3661	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3662	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3663	{
3664	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3665	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3666	{
3667	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3668	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3669	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3670	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3671	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3672	}
3673
3674	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3675	have Unrestricted Execution to handle the guest when it's not using paging. */
3676	GCPhysGuestCR3 = pMixedCtx->cr3;
3677	}
3678	else
3679	{
3680	/*
3681	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3682	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3683	* EPT takes care of translating it to host-physical addresses.
3684	*/
3685	RTGCPHYS GCPhys;
3686	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3687	Assert(PDMVmmDevHeapIsEnabled(pVM));
3688
3689	/* We obtain it here every time as the guest could have relocated this PCI region. */
3690	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3691	AssertRCReturn(rc, rc);
3692
3693	GCPhysGuestCR3 = GCPhys;
3694	}
3695
3696	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3697	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3698	}
3699	else
3700	{
3701	/* Non-nested paging case, just use the hypervisor's CR3. */
3702	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3703
3704	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3705	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3706	}
3707	AssertRCReturn(rc, rc);
3708
3709	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3710	}
3711
3712	/*
3713	* Guest CR4.
3714	*/
3715	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3716	{
3717	Assert(!(pMixedCtx->cr4 >> 32));
3718	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3719
3720	/* The guest's view of its CR4 is unblemished. */
3721	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3722	AssertRCReturn(rc, rc);
3723	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3724
3725	/* Setup VT-x's view of the guest CR4. */
3726	/*
3727	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3728	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3729	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3730	*/
3731	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3732	{
3733	Assert(pVM->hm.s.vmx.pRealModeTSS);
3734	Assert(PDMVmmDevHeapIsEnabled(pVM));
3735	u32GuestCR4 &= ~X86_CR4_VME;
3736	}
3737
3738	if (pVM->hm.s.fNestedPaging)
3739	{
3740	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3741	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3742	{
3743	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3744	u32GuestCR4 \|= X86_CR4_PSE;
3745	/* Our identity mapping is a 32-bit page directory. */
3746	u32GuestCR4 &= ~X86_CR4_PAE;
3747	}
3748	/* else use guest CR4.*/
3749	}
3750	else
3751	{
3752	/*
3753	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3754	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3755	*/
3756	switch (pVCpu->hm.s.enmShadowMode)
3757	{
3758	case PGMMODE_REAL: /* Real-mode. */
3759	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3760	case PGMMODE_32_BIT: /* 32-bit paging. */
3761	{
3762	u32GuestCR4 &= ~X86_CR4_PAE;
3763	break;
3764	}
3765
3766	case PGMMODE_PAE: /* PAE paging. */
3767	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3768	{
3769	u32GuestCR4 \|= X86_CR4_PAE;
3770	break;
3771	}
3772
3773	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3774	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3775	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3776	break;
3777	#endif
3778	default:
3779	AssertFailed();
3780	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3781	}
3782	}
3783
3784	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3785	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3786	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3787	u32GuestCR4 \|= uSetCR4;
3788	u32GuestCR4 &= uZapCR4;
3789
3790	/* Write VT-x's view of the guest CR4 into the VMCS. */
3791	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3792	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3793	AssertRCReturn(rc, rc);
3794
3795	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3796	uint32_t u32CR4Mask = 0;
3797	u32CR4Mask = X86_CR4_VME
3798	\| X86_CR4_PAE
3799	\| X86_CR4_PGE
3800	\| X86_CR4_PSE
3801	\| X86_CR4_VMXE;
3802	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3803	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3804	AssertRCReturn(rc, rc);
3805
3806	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3807	}
3808	return rc;
3809	}
3810
3811
3812	/**
3813	* Loads the guest debug registers into the guest-state area in the VMCS.
3814	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3815	*
3816	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3817	*
3818	* @returns VBox status code.
3819	* @param pVCpu Pointer to the VMCPU.
3820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3821	* out-of-sync. Make sure to update the required fields
3822	* before using them.
3823	*
3824	* @remarks No-long-jump zone!!!
3825	*/
3826	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3827	{
3828	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3829	return VINF_SUCCESS;
3830
3831	#ifdef VBOX_STRICT
3832	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3833	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3834	{
3835	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3836	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3837	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3838	}
3839	#endif
3840
3841	int rc;
3842	PVM pVM = pVCpu->CTX_SUFF(pVM);
3843	bool fInterceptDB = false;
3844	bool fInterceptMovDRx = false;
3845	if ( pVCpu->hm.s.fSingleInstruction
3846	\|\| DBGFIsStepping(pVCpu))
3847	{
3848	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3849	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3850	{
3851	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3852	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3853	AssertRCReturn(rc, rc);
3854	Assert(fInterceptDB == false);
3855	}
3856	else
3857	{
3858	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3859	pVCpu->hm.s.fClearTrapFlag = true;
3860	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3861	fInterceptDB = true;
3862	}
3863	}
3864
3865	if ( fInterceptDB
3866	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3867	{
3868	/*
3869	* Use the combined guest and host DRx values found in the hypervisor
3870	* register set because the debugger has breakpoints active or someone
3871	* is single stepping on the host side without a monitor trap flag.
3872	*
3873	* Note! DBGF expects a clean DR6 state before executing guest code.
3874	*/
3875	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3876	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3877	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3878	{
3879	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3880	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3881	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3882	}
3883	else
3884	#endif
3885	if (!CPUMIsHyperDebugStateActive(pVCpu))
3886	{
3887	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3888	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3889	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3890	}
3891
3892	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3893	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3894	AssertRCReturn(rc, rc);
3895
3896	pVCpu->hm.s.fUsingHyperDR7 = true;
3897	fInterceptDB = true;
3898	fInterceptMovDRx = true;
3899	}
3900	else
3901	{
3902	/*
3903	* If the guest has enabled debug registers, we need to load them prior to
3904	* executing guest code so they'll trigger at the right time.
3905	*/
3906	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3907	{
3908	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3909	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3910	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3911	{
3912	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3913	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3914	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3915	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3916	}
3917	else
3918	#endif
3919	if (!CPUMIsGuestDebugStateActive(pVCpu))
3920	{
3921	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3922	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3923	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3924	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3925	}
3926	Assert(!fInterceptDB);
3927	Assert(!fInterceptMovDRx);
3928	}
3929	/*
3930	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3931	* must intercept #DB in order to maintain a correct DR6 guest value.
3932	*/
3933	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3934	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3935	&& !CPUMIsGuestDebugStateActive(pVCpu))
3936	#else
3937	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3938	#endif
3939	{
3940	fInterceptMovDRx = true;
3941	fInterceptDB = true;
3942	}
3943
3944	/* Update guest DR7. */
3945	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3946	AssertRCReturn(rc, rc);
3947
3948	pVCpu->hm.s.fUsingHyperDR7 = false;
3949	}
3950
3951	/*
3952	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3953	*/
3954	if (fInterceptDB)
3955	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3956	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3957	{
3958	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3959	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3960	#endif
3961	}
3962	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3963	AssertRCReturn(rc, rc);
3964
3965	/*
3966	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3967	*/
3968	if (fInterceptMovDRx)
3969	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3970	else
3971	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3972	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3973	AssertRCReturn(rc, rc);
3974
3975	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3976	return VINF_SUCCESS;
3977	}
3978
3979
3980	#ifdef VBOX_STRICT
3981	/**
3982	* Strict function to validate segment registers.
3983	*
3984	* @remarks ASSUMES CR0 is up to date.
3985	*/
3986	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3987	{
3988	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3989	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3990	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3991	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3992	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3993	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3994	{
3995	/* Protected mode checks */
3996	/* CS */
3997	Assert(pCtx->cs.Attr.n.u1Present);
3998	Assert(!(pCtx->cs.Attr.u & 0xf00));
3999	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4000	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4001	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4002	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4003	\|\| (pCtx->cs.Attr.n.u1Granularity));
4004	/* CS cannot be loaded with NULL in protected mode. */
4005	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4006	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4007	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4008	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4009	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4010	else
4011	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4012	/* SS */
4013	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4014	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4015	if ( !(pCtx->cr0 & X86_CR0_PE)
4016	\|\| pCtx->cs.Attr.n.u4Type == 3)
4017	{
4018	Assert(!pCtx->ss.Attr.n.u2Dpl);
4019	}
4020	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4021	{
4022	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4023	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4024	Assert(pCtx->ss.Attr.n.u1Present);
4025	Assert(!(pCtx->ss.Attr.u & 0xf00));
4026	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4027	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4028	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4029	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4030	\|\| (pCtx->ss.Attr.n.u1Granularity));
4031	}
4032	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4033	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4034	{
4035	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4036	Assert(pCtx->ds.Attr.n.u1Present);
4037	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4038	Assert(!(pCtx->ds.Attr.u & 0xf00));
4039	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4040	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4041	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4042	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4043	\|\| (pCtx->ds.Attr.n.u1Granularity));
4044	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4045	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4046	}
4047	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4048	{
4049	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4050	Assert(pCtx->es.Attr.n.u1Present);
4051	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4052	Assert(!(pCtx->es.Attr.u & 0xf00));
4053	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4054	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4055	\|\| !(pCtx->es.Attr.n.u1Granularity));
4056	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4057	\|\| (pCtx->es.Attr.n.u1Granularity));
4058	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4059	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4060	}
4061	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4062	{
4063	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4064	Assert(pCtx->fs.Attr.n.u1Present);
4065	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4066	Assert(!(pCtx->fs.Attr.u & 0xf00));
4067	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4068	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4069	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4070	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4071	\|\| (pCtx->fs.Attr.n.u1Granularity));
4072	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4073	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4074	}
4075	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4076	{
4077	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4078	Assert(pCtx->gs.Attr.n.u1Present);
4079	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4080	Assert(!(pCtx->gs.Attr.u & 0xf00));
4081	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4082	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4083	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4084	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4085	\|\| (pCtx->gs.Attr.n.u1Granularity));
4086	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4087	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4088	}
4089	/* 64-bit capable CPUs. */
4090	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4091	if (HMVMX_IS_64BIT_HOST_MODE())
4092	{
4093	Assert(!(pCtx->cs.u64Base >> 32));
4094	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4095	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4096	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4097	}
4098	# endif
4099	}
4100	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4101	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4102	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4103	{
4104	/* Real and v86 mode checks. */
4105	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4106	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4107	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4108	{
4109	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4110	}
4111	else
4112	{
4113	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4114	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4115	}
4116
4117	/* CS */
4118	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4119	Assert(pCtx->cs.u32Limit == 0xffff);
4120	Assert(u32CSAttr == 0xf3);
4121	/* SS */
4122	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4123	Assert(pCtx->ss.u32Limit == 0xffff);
4124	Assert(u32SSAttr == 0xf3);
4125	/* DS */
4126	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4127	Assert(pCtx->ds.u32Limit == 0xffff);
4128	Assert(u32DSAttr == 0xf3);
4129	/* ES */
4130	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4131	Assert(pCtx->es.u32Limit == 0xffff);
4132	Assert(u32ESAttr == 0xf3);
4133	/* FS */
4134	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4135	Assert(pCtx->fs.u32Limit == 0xffff);
4136	Assert(u32FSAttr == 0xf3);
4137	/* GS */
4138	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4139	Assert(pCtx->gs.u32Limit == 0xffff);
4140	Assert(u32GSAttr == 0xf3);
4141	/* 64-bit capable CPUs. */
4142	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4143	if (HMVMX_IS_64BIT_HOST_MODE())
4144	{
4145	Assert(!(pCtx->cs.u64Base >> 32));
4146	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4147	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4148	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4149	}
4150	# endif
4151	}
4152	}
4153	#endif /* VBOX_STRICT */
4154
4155
4156	/**
4157	* Writes a guest segment register into the guest-state area in the VMCS.
4158	*
4159	* @returns VBox status code.
4160	* @param pVCpu Pointer to the VMCPU.
4161	* @param idxSel Index of the selector in the VMCS.
4162	* @param idxLimit Index of the segment limit in the VMCS.
4163	* @param idxBase Index of the segment base in the VMCS.
4164	* @param idxAccess Index of the access rights of the segment in the VMCS.
4165	* @param pSelReg Pointer to the segment selector.
4166	*
4167	* @remarks No-long-jump zone!!!
4168	*/
4169	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4170	uint32_t idxAccess, PCPUMSELREG pSelReg)
4171	{
4172	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4173	AssertRCReturn(rc, rc);
4174	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4175	AssertRCReturn(rc, rc);
4176	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4177	AssertRCReturn(rc, rc);
4178
4179	uint32_t u32Access = pSelReg->Attr.u;
4180	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4181	{
4182	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4183	u32Access = 0xf3;
4184	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4185	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4186	}
4187	else
4188	{
4189	/*
4190	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4191	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4192	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4193	* loaded in protected-mode have their attribute as 0.
4194	*/
4195	if (!u32Access)
4196	u32Access = X86DESCATTR_UNUSABLE;
4197	}
4198
4199	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4200	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4201	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4202
4203	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4204	AssertRCReturn(rc, rc);
4205	return rc;
4206	}
4207
4208
4209	/**
4210	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4211	* into the guest-state area in the VMCS.
4212	*
4213	* @returns VBox status code.
4214	* @param pVM Pointer to the VM.
4215	* @param pVCPU Pointer to the VMCPU.
4216	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4217	* out-of-sync. Make sure to update the required fields
4218	* before using them.
4219	*
4220	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4221	* @remarks No-long-jump zone!!!
4222	*/
4223	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4224	{
4225	int rc = VERR_INTERNAL_ERROR_5;
4226	PVM pVM = pVCpu->CTX_SUFF(pVM);
4227
4228	/*
4229	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4230	*/
4231	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4232	{
4233	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4234	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4235	{
4236	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4237	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4238	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4239	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4240	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4241	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4242	}
4243
4244	#ifdef VBOX_WITH_REM
4245	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4246	{
4247	Assert(pVM->hm.s.vmx.pRealModeTSS);
4248	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4249	if ( pVCpu->hm.s.vmx.fWasInRealMode
4250	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4251	{
4252	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4253	in real-mode (e.g. OpenBSD 4.0) */
4254	REMFlushTBs(pVM);
4255	Log4(("Load: Switch to protected mode detected!\n"));
4256	pVCpu->hm.s.vmx.fWasInRealMode = false;
4257	}
4258	}
4259	#endif
4260	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4261	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4262	AssertRCReturn(rc, rc);
4263	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4264	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4265	AssertRCReturn(rc, rc);
4266	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4267	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4268	AssertRCReturn(rc, rc);
4269	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4270	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4271	AssertRCReturn(rc, rc);
4272	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4273	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4274	AssertRCReturn(rc, rc);
4275	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4276	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4277	AssertRCReturn(rc, rc);
4278
4279	#ifdef VBOX_STRICT
4280	/* Validate. */
4281	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4282	#endif
4283
4284	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4285	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4286	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4287	}
4288
4289	/*
4290	* Guest TR.
4291	*/
4292	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4293	{
4294	/*
4295	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4296	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4297	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4298	*/
4299	uint16_t u16Sel = 0;
4300	uint32_t u32Limit = 0;
4301	uint64_t u64Base = 0;
4302	uint32_t u32AccessRights = 0;
4303
4304	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4305	{
4306	u16Sel = pMixedCtx->tr.Sel;
4307	u32Limit = pMixedCtx->tr.u32Limit;
4308	u64Base = pMixedCtx->tr.u64Base;
4309	u32AccessRights = pMixedCtx->tr.Attr.u;
4310	}
4311	else
4312	{
4313	Assert(pVM->hm.s.vmx.pRealModeTSS);
4314	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4315
4316	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4317	RTGCPHYS GCPhys;
4318	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4319	AssertRCReturn(rc, rc);
4320
4321	X86DESCATTR DescAttr;
4322	DescAttr.u = 0;
4323	DescAttr.n.u1Present = 1;
4324	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4325
4326	u16Sel = 0;
4327	u32Limit = HM_VTX_TSS_SIZE;
4328	u64Base = GCPhys; /* in real-mode phys = virt. */
4329	u32AccessRights = DescAttr.u;
4330	}
4331
4332	/* Validate. */
4333	Assert(!(u16Sel & RT_BIT(2)));
4334	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4335	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4336	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4337	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4338	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4339	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4340	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4341	Assert( (u32Limit & 0xfff) == 0xfff
4342	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4343	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4344	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4345
4346	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4347	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4348	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4349	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4350
4351	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4352	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4353	}
4354
4355	/*
4356	* Guest GDTR.
4357	*/
4358	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4359	{
4360	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4361	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4362
4363	/* Validate. */
4364	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4365
4366	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4367	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4368	}
4369
4370	/*
4371	* Guest LDTR.
4372	*/
4373	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4374	{
4375	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4376	uint32_t u32Access = 0;
4377	if (!pMixedCtx->ldtr.Attr.u)
4378	u32Access = X86DESCATTR_UNUSABLE;
4379	else
4380	u32Access = pMixedCtx->ldtr.Attr.u;
4381
4382	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4383	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4384	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4385	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4386
4387	/* Validate. */
4388	if (!(u32Access & X86DESCATTR_UNUSABLE))
4389	{
4390	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4391	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4392	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4393	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4394	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4395	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4396	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4397	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4398	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4399	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4400	}
4401
4402	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4403	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4404	}
4405
4406	/*
4407	* Guest IDTR.
4408	*/
4409	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4410	{
4411	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4412	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4413
4414	/* Validate. */
4415	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4416
4417	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4418	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4419	}
4420
4421	return VINF_SUCCESS;
4422	}
4423
4424
4425	/**
4426	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4427	* areas. These MSRs will automatically be loaded to the host CPU on every
4428	* successful VM entry and stored from the host CPU on every successful VM-exit.
4429	*
4430	* This also creates/updates MSR slots for the host MSRs. The actual host
4431	* MSR values are -not- updated here for performance reasons. See
4432	* hmR0VmxSaveHostMsrs().
4433	*
4434	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4435	*
4436	* @returns VBox status code.
4437	* @param pVCpu Pointer to the VMCPU.
4438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4439	* out-of-sync. Make sure to update the required fields
4440	* before using them.
4441	*
4442	* @remarks No-long-jump zone!!!
4443	*/
4444	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4445	{
4446	AssertPtr(pVCpu);
4447	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4448
4449	/*
4450	* MSRs that we use the auto-load/store MSR area in the VMCS.
4451	*/
4452	PVM pVM = pVCpu->CTX_SUFF(pVM);
4453	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4454	{
4455	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4456	if (pVM->hm.s.fAllow64BitGuests)
4457	{
4458	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4459	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4460	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4461	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4462	# ifdef DEBUG
4463	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4464	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4465	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4466	# endif
4467	}
4468	#endif
4469	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4470	}
4471
4472	/*
4473	* Guest Sysenter MSRs.
4474	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4475	* VM-exits on WRMSRs for these MSRs.
4476	*/
4477	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4478	{
4479	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4480	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4481	}
4482
4483	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4484	{
4485	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4486	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4487	}
4488
4489	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4490	{
4491	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4492	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4493	}
4494
4495	return VINF_SUCCESS;
4496	}
4497
4498
4499	/**
4500	* Loads the guest activity state into the guest-state area in the VMCS.
4501	*
4502	* @returns VBox status code.
4503	* @param pVCpu Pointer to the VMCPU.
4504	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4505	* out-of-sync. Make sure to update the required fields
4506	* before using them.
4507	*
4508	* @remarks No-long-jump zone!!!
4509	*/
4510	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4511	{
4512	NOREF(pCtx);
4513	/** @todo See if we can make use of other states, e.g.
4514	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4515	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4516	{
4517	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4518	AssertRCReturn(rc, rc);
4519
4520	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4521	}
4522	return VINF_SUCCESS;
4523	}
4524
4525
4526	/**
4527	* Sets up the appropriate function to run guest code.
4528	*
4529	* @returns VBox status code.
4530	* @param pVCpu Pointer to the VMCPU.
4531	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4532	* out-of-sync. Make sure to update the required fields
4533	* before using them.
4534	*
4535	* @remarks No-long-jump zone!!!
4536	*/
4537	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4538	{
4539	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4540	{
4541	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4542	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4543	#endif
4544	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4545	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4546	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4547	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4548	{
4549	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4550	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4551	}
4552	#else
4553	/* 64-bit host or hybrid host. */
4554	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4555	#endif
4556	}
4557	else
4558	{
4559	/* Guest is not in long mode, use the 32-bit handler. */
4560	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4561	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4562	{
4563	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4564	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4565	}
4566	#else
4567	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4568	#endif
4569	}
4570	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4571	return VINF_SUCCESS;
4572	}
4573
4574
4575	/**
4576	* Wrapper for running the guest code in VT-x.
4577	*
4578	* @returns VBox strict status code.
4579	* @param pVM Pointer to the VM.
4580	* @param pVCpu Pointer to the VMCPU.
4581	* @param pCtx Pointer to the guest-CPU context.
4582	*
4583	* @remarks No-long-jump zone!!!
4584	*/
4585	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4586	{
4587	/*
4588	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4589	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4590	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4591	*/
4592	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4593	/** @todo Add stats for resume vs launch. */
4594	#ifdef VBOX_WITH_KERNEL_USING_XMM
4595	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4596	#else
4597	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4598	#endif
4599	}
4600
4601
4602	/**
4603	* Reports world-switch error and dumps some useful debug info.
4604	*
4605	* @param pVM Pointer to the VM.
4606	* @param pVCpu Pointer to the VMCPU.
4607	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4608	* @param pCtx Pointer to the guest-CPU context.
4609	* @param pVmxTransient Pointer to the VMX transient structure (only
4610	* exitReason updated).
4611	*/
4612	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4613	{
4614	Assert(pVM);
4615	Assert(pVCpu);
4616	Assert(pCtx);
4617	Assert(pVmxTransient);
4618	HMVMX_ASSERT_PREEMPT_SAFE();
4619
4620	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4621	switch (rcVMRun)
4622	{
4623	case VERR_VMX_INVALID_VMXON_PTR:
4624	AssertFailed();
4625	break;
4626	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4627	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4628	{
4629	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4630	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4631	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4632	AssertRC(rc);
4633
4634	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4635	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4636	Cannot do it here as we may have been long preempted. */
4637
4638	#ifdef VBOX_STRICT
4639	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4640	pVmxTransient->uExitReason));
4641	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4642	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4643	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4644	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4645	else
4646	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4647	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4648	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4649
4650	/* VMX control bits. */
4651	uint32_t u32Val;
4652	uint64_t u64Val;
4653	HMVMXHCUINTREG uHCReg;
4654	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4655	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4656	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4657	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4658	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4659	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4660	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4661	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4663	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4665	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4667	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4668	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4669	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4671	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4673	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4675	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4677	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4679	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4686	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4687	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4688	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4689	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4690	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4691	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4693	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4694	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4695	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4696
4697	/* Guest bits. */
4698	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4699	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4700	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4701	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4702	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4703	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4704	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4705	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4706
4707	/* Host bits. */
4708	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4709	Log4(("Host CR0 %#RHr\n", uHCReg));
4710	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4711	Log4(("Host CR3 %#RHr\n", uHCReg));
4712	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4713	Log4(("Host CR4 %#RHr\n", uHCReg));
4714
4715	RTGDTR HostGdtr;
4716	PCX86DESCHC pDesc;
4717	ASMGetGDTR(&HostGdtr);
4718	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4719	Log4(("Host CS %#08x\n", u32Val));
4720	if (u32Val < HostGdtr.cbGdt)
4721	{
4722	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4723	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4724	}
4725
4726	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4727	Log4(("Host DS %#08x\n", u32Val));
4728	if (u32Val < HostGdtr.cbGdt)
4729	{
4730	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4731	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4732	}
4733
4734	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4735	Log4(("Host ES %#08x\n", u32Val));
4736	if (u32Val < HostGdtr.cbGdt)
4737	{
4738	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4739	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4740	}
4741
4742	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4743	Log4(("Host FS %#08x\n", u32Val));
4744	if (u32Val < HostGdtr.cbGdt)
4745	{
4746	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4747	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4748	}
4749
4750	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4751	Log4(("Host GS %#08x\n", u32Val));
4752	if (u32Val < HostGdtr.cbGdt)
4753	{
4754	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4755	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4756	}
4757
4758	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4759	Log4(("Host SS %#08x\n", u32Val));
4760	if (u32Val < HostGdtr.cbGdt)
4761	{
4762	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4763	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4764	}
4765
4766	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4767	Log4(("Host TR %#08x\n", u32Val));
4768	if (u32Val < HostGdtr.cbGdt)
4769	{
4770	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4771	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4772	}
4773
4774	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4775	Log4(("Host TR Base %#RHv\n", uHCReg));
4776	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4777	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4778	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4779	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4780	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4781	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4782	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4783	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4784	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4785	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4786	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4787	Log4(("Host RSP %#RHv\n", uHCReg));
4788	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4789	Log4(("Host RIP %#RHv\n", uHCReg));
4790	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4791	if (HMVMX_IS_64BIT_HOST_MODE())
4792	{
4793	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4794	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4795	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4796	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4797	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4798	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4799	}
4800	# endif
4801	#endif /* VBOX_STRICT */
4802	break;
4803	}
4804
4805	default:
4806	/* Impossible */
4807	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4808	break;
4809	}
4810	NOREF(pVM); NOREF(pCtx);
4811	}
4812
4813
4814	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4815	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4816	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4817	#endif
4818	#ifdef VBOX_STRICT
4819	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4820	{
4821	switch (idxField)
4822	{
4823	case VMX_VMCS_GUEST_RIP:
4824	case VMX_VMCS_GUEST_RSP:
4825	case VMX_VMCS_GUEST_SYSENTER_EIP:
4826	case VMX_VMCS_GUEST_SYSENTER_ESP:
4827	case VMX_VMCS_GUEST_GDTR_BASE:
4828	case VMX_VMCS_GUEST_IDTR_BASE:
4829	case VMX_VMCS_GUEST_CS_BASE:
4830	case VMX_VMCS_GUEST_DS_BASE:
4831	case VMX_VMCS_GUEST_ES_BASE:
4832	case VMX_VMCS_GUEST_FS_BASE:
4833	case VMX_VMCS_GUEST_GS_BASE:
4834	case VMX_VMCS_GUEST_SS_BASE:
4835	case VMX_VMCS_GUEST_LDTR_BASE:
4836	case VMX_VMCS_GUEST_TR_BASE:
4837	case VMX_VMCS_GUEST_CR3:
4838	return true;
4839	}
4840	return false;
4841	}
4842
4843	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4844	{
4845	switch (idxField)
4846	{
4847	/* Read-only fields. */
4848	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4849	return true;
4850	}
4851	/* Remaining readable fields should also be writable. */
4852	return hmR0VmxIsValidWriteField(idxField);
4853	}
4854	#endif /* VBOX_STRICT */
4855
4856
4857	/**
4858	* Executes the specified handler in 64-bit mode.
4859	*
4860	* @returns VBox status code.
4861	* @param pVM Pointer to the VM.
4862	* @param pVCpu Pointer to the VMCPU.
4863	* @param pCtx Pointer to the guest CPU context.
4864	* @param enmOp The operation to perform.
4865	* @param cbParam Number of parameters.
4866	* @param paParam Array of 32-bit parameters.
4867	*/
4868	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4869	uint32_t *paParam)
4870	{
4871	int rc, rc2;
4872	PHMGLOBALCPUINFO pCpu;
4873	RTHCPHYS HCPhysCpuPage;
4874	RTCCUINTREG uOldEflags;
4875
4876	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4877	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4878	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4879	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4880
4881	#ifdef VBOX_STRICT
4882	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4883	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4884
4885	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4886	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4887	#endif
4888
4889	/* Disable interrupts. */
4890	uOldEflags = ASMIntDisableFlags();
4891
4892	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4893	RTCPUID idHostCpu = RTMpCpuId();
4894	CPUMR0SetLApic(pVCpu, idHostCpu);
4895	#endif
4896
4897	pCpu = HMR0GetCurrentCpu();
4898	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4899
4900	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4901	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4902
4903	/* Leave VMX Root Mode. */
4904	VMXDisable();
4905
4906	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4907
4908	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4909	CPUMSetHyperEIP(pVCpu, enmOp);
4910	for (int i = (int)cbParam - 1; i >= 0; i--)
4911	CPUMPushHyper(pVCpu, paParam[i]);
4912
4913	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4914
4915	/* Call the switcher. */
4916	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4917	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4918
4919	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4920	/* Make sure the VMX instructions don't cause #UD faults. */
4921	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4922
4923	/* Re-enter VMX Root Mode */
4924	rc2 = VMXEnable(HCPhysCpuPage);
4925	if (RT_FAILURE(rc2))
4926	{
4927	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4928	ASMSetFlags(uOldEflags);
4929	return rc2;
4930	}
4931
4932	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4933	AssertRC(rc2);
4934	Assert(!(ASMGetFlags() & X86_EFL_IF));
4935	ASMSetFlags(uOldEflags);
4936	return rc;
4937	}
4938
4939
4940	/**
4941	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4942	* supporting 64-bit guests.
4943	*
4944	* @returns VBox status code.
4945	* @param fResume Whether to VMLAUNCH or VMRESUME.
4946	* @param pCtx Pointer to the guest-CPU context.
4947	* @param pCache Pointer to the VMCS cache.
4948	* @param pVM Pointer to the VM.
4949	* @param pVCpu Pointer to the VMCPU.
4950	*/
4951	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4952	{
4953	uint32_t aParam[6];
4954	PHMGLOBALCPUINFO pCpu = NULL;
4955	RTHCPHYS HCPhysCpuPage = 0;
4956	int rc = VERR_INTERNAL_ERROR_5;
4957
4958	pCpu = HMR0GetCurrentCpu();
4959	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4960
4961	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4962	pCache->uPos = 1;
4963	pCache->interPD = PGMGetInterPaeCR3(pVM);
4964	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4965	#endif
4966
4967	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4968	pCache->TestIn.HCPhysCpuPage = 0;
4969	pCache->TestIn.HCPhysVmcs = 0;
4970	pCache->TestIn.pCache = 0;
4971	pCache->TestOut.HCPhysVmcs = 0;
4972	pCache->TestOut.pCache = 0;
4973	pCache->TestOut.pCtx = 0;
4974	pCache->TestOut.eflags = 0;
4975	#endif
4976
4977	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4978	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4979	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4980	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4981	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4982	aParam[5] = 0;
4983
4984	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4985	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4986	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4987	#endif
4988	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4989
4990	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4991	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4992	Assert(pCtx->dr[4] == 10);
4993	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4994	#endif
4995
4996	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4997	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4998	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4999	pVCpu->hm.s.vmx.HCPhysVmcs));
5000	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5001	pCache->TestOut.HCPhysVmcs));
5002	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5003	pCache->TestOut.pCache));
5004	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5005	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5006	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5007	pCache->TestOut.pCtx));
5008	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5009	#endif
5010	return rc;
5011	}
5012
5013
5014	/**
5015	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5016	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5017	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5018	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5019	*
5020	* @returns VBox status code.
5021	* @param pVM Pointer to the VM.
5022	* @param pVCpu Pointer to the VMCPU.
5023	*/
5024	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5025	{
5026	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5027	{ \
5028	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5029	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5030	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5031	++cReadFields; \
5032	}
5033
5034	AssertPtr(pVM);
5035	AssertPtr(pVCpu);
5036	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5037	uint32_t cReadFields = 0;
5038
5039	/*
5040	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5041	* and serve to indicate exceptions to the rules.
5042	*/
5043
5044	/* Guest-natural selector base fields. */
5045	#if 0
5046	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5047	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5048	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5049	#endif
5050	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5051	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5052	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5053	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5055	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5062	#if 0
5063	/* Unused natural width guest-state fields. */
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5066	#endif
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5069
5070	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5071	#if 0
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5081	#endif
5082
5083	/* Natural width guest-state fields. */
5084	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5085	#if 0
5086	/* Currently unused field. */
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5088	#endif
5089
5090	if (pVM->hm.s.fNestedPaging)
5091	{
5092	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5093	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5094	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5095	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5096	}
5097	else
5098	{
5099	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5100	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5101	}
5102
5103	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5104	return VINF_SUCCESS;
5105	}
5106
5107
5108	/**
5109	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5110	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5111	* darwin, running 64-bit guests).
5112	*
5113	* @returns VBox status code.
5114	* @param pVCpu Pointer to the VMCPU.
5115	* @param idxField The VMCS field encoding.
5116	* @param u64Val 16, 32 or 64-bit value.
5117	*/
5118	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5119	{
5120	int rc;
5121	switch (idxField)
5122	{
5123	/*
5124	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5125	*/
5126	/* 64-bit Control fields. */
5127	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5128	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5129	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5130	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5131	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5132	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5133	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5134	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5135	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5136	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5137	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5138	case VMX_VMCS64_CTRL_EPTP_FULL:
5139	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5140	/* 64-bit Guest-state fields. */
5141	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5142	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5143	case VMX_VMCS64_GUEST_PAT_FULL:
5144	case VMX_VMCS64_GUEST_EFER_FULL:
5145	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5146	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5147	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5148	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5149	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5150	/* 64-bit Host-state fields. */
5151	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5152	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5153	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5154	{
5155	rc = VMXWriteVmcs32(idxField, u64Val);
5156	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5157	break;
5158	}
5159
5160	/*
5161	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5162	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5163	*/
5164	/* Natural-width Guest-state fields. */
5165	case VMX_VMCS_GUEST_CR3:
5166	case VMX_VMCS_GUEST_ES_BASE:
5167	case VMX_VMCS_GUEST_CS_BASE:
5168	case VMX_VMCS_GUEST_SS_BASE:
5169	case VMX_VMCS_GUEST_DS_BASE:
5170	case VMX_VMCS_GUEST_FS_BASE:
5171	case VMX_VMCS_GUEST_GS_BASE:
5172	case VMX_VMCS_GUEST_LDTR_BASE:
5173	case VMX_VMCS_GUEST_TR_BASE:
5174	case VMX_VMCS_GUEST_GDTR_BASE:
5175	case VMX_VMCS_GUEST_IDTR_BASE:
5176	case VMX_VMCS_GUEST_RSP:
5177	case VMX_VMCS_GUEST_RIP:
5178	case VMX_VMCS_GUEST_SYSENTER_ESP:
5179	case VMX_VMCS_GUEST_SYSENTER_EIP:
5180	{
5181	if (!(u64Val >> 32))
5182	{
5183	/* If this field is 64-bit, VT-x will zero out the top bits. */
5184	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5185	}
5186	else
5187	{
5188	/* Assert that only the 32->64 switcher case should ever come here. */
5189	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5190	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5191	}
5192	break;
5193	}
5194
5195	default:
5196	{
5197	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5198	rc = VERR_INVALID_PARAMETER;
5199	break;
5200	}
5201	}
5202	AssertRCReturn(rc, rc);
5203	return rc;
5204	}
5205
5206
5207	/**
5208	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5209	* hosts (except darwin) for 64-bit guests.
5210	*
5211	* @param pVCpu Pointer to the VMCPU.
5212	* @param idxField The VMCS field encoding.
5213	* @param u64Val 16, 32 or 64-bit value.
5214	*/
5215	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5216	{
5217	AssertPtr(pVCpu);
5218	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5219
5220	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5221	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5222
5223	/* Make sure there are no duplicates. */
5224	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5225	{
5226	if (pCache->Write.aField[i] == idxField)
5227	{
5228	pCache->Write.aFieldVal[i] = u64Val;
5229	return VINF_SUCCESS;
5230	}
5231	}
5232
5233	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5234	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5235	pCache->Write.cValidEntries++;
5236	return VINF_SUCCESS;
5237	}
5238
5239	/* Enable later when the assembly code uses these as callbacks. */
5240	#if 0
5241	/*
5242	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5243	*
5244	* @param pVCpu Pointer to the VMCPU.
5245	* @param pCache Pointer to the VMCS cache.
5246	*
5247	* @remarks No-long-jump zone!!!
5248	*/
5249	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5250	{
5251	AssertPtr(pCache);
5252	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5253	{
5254	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5255	AssertRC(rc);
5256	}
5257	pCache->Write.cValidEntries = 0;
5258	}
5259
5260
5261	/**
5262	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5263	*
5264	* @param pVCpu Pointer to the VMCPU.
5265	* @param pCache Pointer to the VMCS cache.
5266	*
5267	* @remarks No-long-jump zone!!!
5268	*/
5269	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5270	{
5271	AssertPtr(pCache);
5272	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5273	{
5274	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5275	AssertRC(rc);
5276	}
5277	}
5278	#endif
5279	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5280
5281
5282	/**
5283	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5284	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5285	* timer.
5286	*
5287	* @returns VBox status code.
5288	* @param pVCpu Pointer to the VMCPU.
5289	*
5290	* @remarks No-long-jump zone!!!
5291	*/
5292	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5293	{
5294	int rc = VERR_INTERNAL_ERROR_5;
5295	bool fOffsettedTsc = false;
5296	PVM pVM = pVCpu->CTX_SUFF(pVM);
5297	if (pVM->hm.s.vmx.fUsePreemptTimer)
5298	{
5299	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5300
5301	/* Make sure the returned values have sane upper and lower boundaries. */
5302	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5303	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5304	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5305	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5306
5307	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5308	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5309	}
5310	else
5311	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5312
5313	if (fOffsettedTsc)
5314	{
5315	uint64_t u64CurTSC = ASMReadTSC();
5316	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5317	{
5318	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5319	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5320
5321	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5322	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5323	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5324	}
5325	else
5326	{
5327	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5328	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5329	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5331	}
5332	}
5333	else
5334	{
5335	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5336	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5337	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5338	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5339	}
5340	}
5341
5342
5343	/**
5344	* Determines if an exception is a contributory exception. Contributory
5345	* exceptions are ones which can cause double-faults. Page-fault is
5346	* intentionally not included here as it's a conditional contributory exception.
5347	*
5348	* @returns true if the exception is contributory, false otherwise.
5349	* @param uVector The exception vector.
5350	*/
5351	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5352	{
5353	switch (uVector)
5354	{
5355	case X86_XCPT_GP:
5356	case X86_XCPT_SS:
5357	case X86_XCPT_NP:
5358	case X86_XCPT_TS:
5359	case X86_XCPT_DE:
5360	return true;
5361	default:
5362	break;
5363	}
5364	return false;
5365	}
5366
5367
5368	/**
5369	* Sets an event as a pending event to be injected into the guest.
5370	*
5371	* @param pVCpu Pointer to the VMCPU.
5372	* @param u32IntInfo The VM-entry interruption-information field.
5373	* @param cbInstr The VM-entry instruction length in bytes (for software
5374	* interrupts, exceptions and privileged software
5375	* exceptions).
5376	* @param u32ErrCode The VM-entry exception error code.
5377	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5378	* page-fault.
5379	*
5380	* @remarks Statistics counter assumes this is a guest event being injected or
5381	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5382	* always incremented.
5383	*/
5384	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5385	RTGCUINTPTR GCPtrFaultAddress)
5386	{
5387	Assert(!pVCpu->hm.s.Event.fPending);
5388	pVCpu->hm.s.Event.fPending = true;
5389	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5390	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5391	pVCpu->hm.s.Event.cbInstr = cbInstr;
5392	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5393
5394	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5395	}
5396
5397
5398	/**
5399	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5400	*
5401	* @param pVCpu Pointer to the VMCPU.
5402	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5403	* out-of-sync. Make sure to update the required fields
5404	* before using them.
5405	*/
5406	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5407	{
5408	NOREF(pMixedCtx);
5409	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5410	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5411	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5412	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5413	}
5414
5415
5416	/**
5417	* Handle a condition that occurred while delivering an event through the guest
5418	* IDT.
5419	*
5420	* @returns VBox status code (informational error codes included).
5421	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5422	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5423	* continue execution of the guest which will delivery the #DF.
5424	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5425	*
5426	* @param pVCpu Pointer to the VMCPU.
5427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5428	* out-of-sync. Make sure to update the required fields
5429	* before using them.
5430	* @param pVmxTransient Pointer to the VMX transient structure.
5431	*
5432	* @remarks No-long-jump zone!!!
5433	*/
5434	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5435	{
5436	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5437	AssertRCReturn(rc, rc);
5438	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5439	{
5440	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5441	AssertRCReturn(rc, rc);
5442
5443	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5444	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5445	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5446
5447	typedef enum
5448	{
5449	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5450	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5451	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5452	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5453	} VMXREFLECTXCPT;
5454
5455	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5456	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5457	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5458	{
5459	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5460	{
5461	enmReflect = VMXREFLECTXCPT_XCPT;
5462	#ifdef VBOX_STRICT
5463	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5464	&& uExitVector == X86_XCPT_PF)
5465	{
5466	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5467	}
5468	#endif
5469	if ( uExitVector == X86_XCPT_PF
5470	&& uIdtVector == X86_XCPT_PF)
5471	{
5472	pVmxTransient->fVectoringPF = true;
5473	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5474	}
5475	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5476	&& hmR0VmxIsContributoryXcpt(uExitVector)
5477	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5478	\|\| uIdtVector == X86_XCPT_PF))
5479	{
5480	enmReflect = VMXREFLECTXCPT_DF;
5481	}
5482	else if (uIdtVector == X86_XCPT_DF)
5483	enmReflect = VMXREFLECTXCPT_TF;
5484	}
5485	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5486	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5487	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5488	{
5489	/*
5490	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5491	* (whatever they are) as they reoccur when restarting the instruction.
5492	*/
5493	enmReflect = VMXREFLECTXCPT_XCPT;
5494	}
5495	}
5496	else
5497	{
5498	/*
5499	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5500	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5501	* original exception to the guest after handling the VM-exit.
5502	*/
5503	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5504	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5505	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5506	{
5507	enmReflect = VMXREFLECTXCPT_XCPT;
5508	}
5509	}
5510
5511	switch (enmReflect)
5512	{
5513	case VMXREFLECTXCPT_XCPT:
5514	{
5515	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5516	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5517	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5518
5519	uint32_t u32ErrCode = 0;
5520	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5521	{
5522	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5523	AssertRCReturn(rc, rc);
5524	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5525	}
5526
5527	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5528	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5529	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5530	rc = VINF_SUCCESS;
5531	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5532	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5533
5534	break;
5535	}
5536
5537	case VMXREFLECTXCPT_DF:
5538	{
5539	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5540	rc = VINF_HM_DOUBLE_FAULT;
5541	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5542	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5543
5544	break;
5545	}
5546
5547	case VMXREFLECTXCPT_TF:
5548	{
5549	rc = VINF_EM_RESET;
5550	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5551	uExitVector));
5552	break;
5553	}
5554
5555	default:
5556	Assert(rc == VINF_SUCCESS);
5557	break;
5558	}
5559	}
5560	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5561	return rc;
5562	}
5563
5564
5565	/**
5566	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5567	*
5568	* @returns VBox status code.
5569	* @param pVCpu Pointer to the VMCPU.
5570	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5571	* out-of-sync. Make sure to update the required fields
5572	* before using them.
5573	*
5574	* @remarks No-long-jump zone!!!
5575	*/
5576	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5577	{
5578	NOREF(pMixedCtx);
5579
5580	/* Since this can be called from our preemption hook it's safer to make the guest-CR0 update non-preemptible. */
5581	VMMRZCallRing3Disable(pVCpu);
5582	HM_DISABLE_PREEMPT_IF_NEEDED();
5583
5584	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5585	{
5586	uint32_t uVal = 0;
5587	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5588	AssertRCReturn(rc, rc);
5589
5590	uint32_t uShadow = 0;
5591	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5592	AssertRCReturn(rc, rc);
5593
5594	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5595	CPUMSetGuestCR0(pVCpu, uVal);
5596	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5597	}
5598
5599	HM_RESTORE_PREEMPT_IF_NEEDED();
5600	VMMRZCallRing3Enable(pVCpu);
5601
5602	return VINF_SUCCESS;
5603	}
5604
5605
5606	/**
5607	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5608	*
5609	* @returns VBox status code.
5610	* @param pVCpu Pointer to the VMCPU.
5611	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5612	* out-of-sync. Make sure to update the required fields
5613	* before using them.
5614	*
5615	* @remarks No-long-jump zone!!!
5616	*/
5617	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5618	{
5619	NOREF(pMixedCtx);
5620
5621	int rc = VINF_SUCCESS;
5622	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5623	{
5624	uint32_t uVal = 0;
5625	uint32_t uShadow = 0;
5626	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5627	AssertRCReturn(rc, rc);
5628	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5629	AssertRCReturn(rc, rc);
5630
5631	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5632	CPUMSetGuestCR4(pVCpu, uVal);
5633	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5634	}
5635	return rc;
5636	}
5637
5638
5639	/**
5640	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5641	*
5642	* @returns VBox status code.
5643	* @param pVCpu Pointer to the VMCPU.
5644	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5645	* out-of-sync. Make sure to update the required fields
5646	* before using them.
5647	*
5648	* @remarks No-long-jump zone!!!
5649	*/
5650	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5651	{
5652	int rc = VINF_SUCCESS;
5653	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5654	{
5655	uint64_t u64Val = 0;
5656	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5657	AssertRCReturn(rc, rc);
5658
5659	pMixedCtx->rip = u64Val;
5660	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5661	}
5662	return rc;
5663	}
5664
5665
5666	/**
5667	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5668	*
5669	* @returns VBox status code.
5670	* @param pVCpu Pointer to the VMCPU.
5671	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5672	* out-of-sync. Make sure to update the required fields
5673	* before using them.
5674	*
5675	* @remarks No-long-jump zone!!!
5676	*/
5677	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5678	{
5679	int rc = VINF_SUCCESS;
5680	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5681	{
5682	uint64_t u64Val = 0;
5683	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5684	AssertRCReturn(rc, rc);
5685
5686	pMixedCtx->rsp = u64Val;
5687	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5688	}
5689	return rc;
5690	}
5691
5692
5693	/**
5694	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5695	*
5696	* @returns VBox status code.
5697	* @param pVCpu Pointer to the VMCPU.
5698	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5699	* out-of-sync. Make sure to update the required fields
5700	* before using them.
5701	*
5702	* @remarks No-long-jump zone!!!
5703	*/
5704	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5705	{
5706	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5707	{
5708	uint32_t uVal = 0;
5709	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5710	AssertRCReturn(rc, rc);
5711
5712	pMixedCtx->eflags.u32 = uVal;
5713	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5714	{
5715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5716	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5717
5718	pMixedCtx->eflags.Bits.u1VM = 0;
5719	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5720	}
5721
5722	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5723	}
5724	return VINF_SUCCESS;
5725	}
5726
5727
5728	/**
5729	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5730	* guest-CPU context.
5731	*/
5732	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5733	{
5734	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5735	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5736	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5737	return rc;
5738	}
5739
5740
5741	/**
5742	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5743	* from the guest-state area in the VMCS.
5744	*
5745	* @param pVCpu Pointer to the VMCPU.
5746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5747	* out-of-sync. Make sure to update the required fields
5748	* before using them.
5749	*
5750	* @remarks No-long-jump zone!!!
5751	*/
5752	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753	{
5754	uint32_t uIntrState = 0;
5755	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5756	AssertRC(rc);
5757
5758	if (!uIntrState)
5759	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5760	else
5761	{
5762	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5763	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5764	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5765	AssertRC(rc);
5766	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5767	AssertRC(rc);
5768
5769	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5770	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5771	}
5772	}
5773
5774
5775	/**
5776	* Saves the guest's activity state.
5777	*
5778	* @returns VBox status code.
5779	* @param pVCpu Pointer to the VMCPU.
5780	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5781	* out-of-sync. Make sure to update the required fields
5782	* before using them.
5783	*
5784	* @remarks No-long-jump zone!!!
5785	*/
5786	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5787	{
5788	NOREF(pMixedCtx);
5789	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5790	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5791	return VINF_SUCCESS;
5792	}
5793
5794
5795	/**
5796	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5797	* the current VMCS into the guest-CPU context.
5798	*
5799	* @returns VBox status code.
5800	* @param pVCpu Pointer to the VMCPU.
5801	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5802	* out-of-sync. Make sure to update the required fields
5803	* before using them.
5804	*
5805	* @remarks No-long-jump zone!!!
5806	*/
5807	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5808	{
5809	int rc = VINF_SUCCESS;
5810	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5811	{
5812	uint32_t u32Val = 0;
5813	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5814	pMixedCtx->SysEnter.cs = u32Val;
5815	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5816	}
5817
5818	uint64_t u64Val = 0;
5819	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5820	{
5821	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5822	pMixedCtx->SysEnter.eip = u64Val;
5823	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5824	}
5825	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5826	{
5827	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5828	pMixedCtx->SysEnter.esp = u64Val;
5829	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5830	}
5831	return rc;
5832	}
5833
5834
5835	/**
5836	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5837	* the CPU back into the guest-CPU context.
5838	*
5839	* @returns VBox status code.
5840	* @param pVCpu Pointer to the VMCPU.
5841	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5842	* out-of-sync. Make sure to update the required fields
5843	* before using them.
5844	*
5845	* @remarks No-long-jump zone!!!
5846	*/
5847	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5848	{
5849	#if HC_ARCH_BITS == 64
5850	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5851	{
5852	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5853	VMMRZCallRing3Disable(pVCpu);
5854	HM_DISABLE_PREEMPT_IF_NEEDED();
5855
5856	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5857	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5858	{
5859	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5860	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5861	}
5862
5863	HM_RESTORE_PREEMPT_IF_NEEDED();
5864	VMMRZCallRing3Enable(pVCpu);
5865	}
5866	else
5867	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5868	#else
5869	NOREF(pMixedCtx);
5870	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5871	#endif
5872
5873	return VINF_SUCCESS;
5874	}
5875
5876
5877	/**
5878	* Saves the auto load/store'd guest MSRs from the current VMCS into
5879	* the guest-CPU context.
5880	*
5881	* @returns VBox status code.
5882	* @param pVCpu Pointer to the VMCPU.
5883	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5884	* out-of-sync. Make sure to update the required fields
5885	* before using them.
5886	*
5887	* @remarks No-long-jump zone!!!
5888	*/
5889	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890	{
5891	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5892	return VINF_SUCCESS;
5893
5894	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5895	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5896	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5897	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5898	{
5899	switch (pMsr->u32Msr)
5900	{
5901	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5902	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5903	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5904	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5905	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5906	default:
5907	{
5908	AssertFailed();
5909	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5910	}
5911	}
5912	}
5913
5914	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5915	return VINF_SUCCESS;
5916	}
5917
5918
5919	/**
5920	* Saves the guest control registers from the current VMCS into the guest-CPU
5921	* context.
5922	*
5923	* @returns VBox status code.
5924	* @param pVCpu Pointer to the VMCPU.
5925	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5926	* out-of-sync. Make sure to update the required fields
5927	* before using them.
5928	*
5929	* @remarks No-long-jump zone!!!
5930	*/
5931	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5932	{
5933	/* Guest CR0. Guest FPU. */
5934	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5935	AssertRCReturn(rc, rc);
5936
5937	/* Guest CR4. */
5938	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5939	AssertRCReturn(rc, rc);
5940
5941	/* Guest CR2 - updated always during the world-switch or in #PF. */
5942	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5943	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5944	{
5945	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5946	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5947
5948	PVM pVM = pVCpu->CTX_SUFF(pVM);
5949	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5950	\|\| ( pVM->hm.s.fNestedPaging
5951	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5952	{
5953	uint64_t u64Val = 0;
5954	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5955	if (pMixedCtx->cr3 != u64Val)
5956	{
5957	CPUMSetGuestCR3(pVCpu, u64Val);
5958	if (VMMRZCallRing3IsEnabled(pVCpu))
5959	{
5960	PGMUpdateCR3(pVCpu, u64Val);
5961	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5962	}
5963	else
5964	{
5965	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5966	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5967	}
5968	}
5969
5970	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5971	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5972	{
5973	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5974	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5975	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5976	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5977
5978	if (VMMRZCallRing3IsEnabled(pVCpu))
5979	{
5980	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5981	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5982	}
5983	else
5984	{
5985	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5986	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5987	}
5988	}
5989	}
5990
5991	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
5992	}
5993
5994	/*
5995	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5996	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5997	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5998	*
5999	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6000	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6001	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6002	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6003	*
6004	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6005	*/
6006	if (VMMRZCallRing3IsEnabled(pVCpu))
6007	{
6008	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6009	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6010
6011	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6012	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6013
6014	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6015	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6016	}
6017
6018	return rc;
6019	}
6020
6021
6022	/**
6023	* Reads a guest segment register from the current VMCS into the guest-CPU
6024	* context.
6025	*
6026	* @returns VBox status code.
6027	* @param pVCpu Pointer to the VMCPU.
6028	* @param idxSel Index of the selector in the VMCS.
6029	* @param idxLimit Index of the segment limit in the VMCS.
6030	* @param idxBase Index of the segment base in the VMCS.
6031	* @param idxAccess Index of the access rights of the segment in the VMCS.
6032	* @param pSelReg Pointer to the segment selector.
6033	*
6034	* @remarks No-long-jump zone!!!
6035	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6036	* macro as that takes care of whether to read from the VMCS cache or
6037	* not.
6038	*/
6039	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6040	PCPUMSELREG pSelReg)
6041	{
6042	NOREF(pVCpu);
6043
6044	uint32_t u32Val = 0;
6045	int rc = VMXReadVmcs32(idxSel, &u32Val);
6046	AssertRCReturn(rc, rc);
6047	pSelReg->Sel = (uint16_t)u32Val;
6048	pSelReg->ValidSel = (uint16_t)u32Val;
6049	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6050
6051	rc = VMXReadVmcs32(idxLimit, &u32Val);
6052	AssertRCReturn(rc, rc);
6053	pSelReg->u32Limit = u32Val;
6054
6055	uint64_t u64Val = 0;
6056	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6057	AssertRCReturn(rc, rc);
6058	pSelReg->u64Base = u64Val;
6059
6060	rc = VMXReadVmcs32(idxAccess, &u32Val);
6061	AssertRCReturn(rc, rc);
6062	pSelReg->Attr.u = u32Val;
6063
6064	/*
6065	* If VT-x marks the segment as unusable, most other bits remain undefined:
6066	* - For CS the L, D and G bits have meaning.
6067	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6068	* - For the remaining data segments no bits are defined.
6069	*
6070	* The present bit and the unusable bit has been observed to be set at the
6071	* same time (the selector was supposed to invalid as we started executing
6072	* a V8086 interrupt in ring-0).
6073	*
6074	* What should be important for the rest of the VBox code, is that the P bit is
6075	* cleared. Some of the other VBox code recognizes the unusable bit, but
6076	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6077	* safe side here, we'll strip off P and other bits we don't care about. If
6078	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6079	*
6080	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6081	*/
6082	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6083	{
6084	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6085
6086	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6087	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6088	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6089
6090	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6091	#ifdef DEBUG_bird
6092	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6093	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6094	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6095	#endif
6096	}
6097	return VINF_SUCCESS;
6098	}
6099
6100
6101	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6102	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6103	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6104	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6105	#else
6106	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6107	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6108	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6109	#endif
6110
6111
6112	/**
6113	* Saves the guest segment registers from the current VMCS into the guest-CPU
6114	* context.
6115	*
6116	* @returns VBox status code.
6117	* @param pVCpu Pointer to the VMCPU.
6118	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6119	* out-of-sync. Make sure to update the required fields
6120	* before using them.
6121	*
6122	* @remarks No-long-jump zone!!!
6123	*/
6124	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6125	{
6126	/* Guest segment registers. */
6127	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6128	{
6129	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6130	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6131	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6132	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6133	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6134	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6135	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6136
6137	/* Restore segment attributes for real-on-v86 mode hack. */
6138	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6139	{
6140	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6141	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6142	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6143	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6144	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6145	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6146	}
6147	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6148	}
6149
6150	return VINF_SUCCESS;
6151	}
6152
6153
6154	/**
6155	* Saves the guest descriptor table registers and task register from the current
6156	* VMCS into the guest-CPU context.
6157	*
6158	* @returns VBox status code.
6159	* @param pVCpu Pointer to the VMCPU.
6160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6161	* out-of-sync. Make sure to update the required fields
6162	* before using them.
6163	*
6164	* @remarks No-long-jump zone!!!
6165	*/
6166	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6167	{
6168	int rc = VINF_SUCCESS;
6169
6170	/* Guest LDTR. */
6171	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6172	{
6173	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6174	AssertRCReturn(rc, rc);
6175	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6176	}
6177
6178	/* Guest GDTR. */
6179	uint64_t u64Val = 0;
6180	uint32_t u32Val = 0;
6181	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6182	{
6183	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6184	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6185	pMixedCtx->gdtr.pGdt = u64Val;
6186	pMixedCtx->gdtr.cbGdt = u32Val;
6187	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6188	}
6189
6190	/* Guest IDTR. */
6191	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6192	{
6193	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6194	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6195	pMixedCtx->idtr.pIdt = u64Val;
6196	pMixedCtx->idtr.cbIdt = u32Val;
6197	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6198	}
6199
6200	/* Guest TR. */
6201	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6202	{
6203	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6204	AssertRCReturn(rc, rc);
6205
6206	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6207	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6208	{
6209	rc = VMXLOCAL_READ_SEG(TR, tr);
6210	AssertRCReturn(rc, rc);
6211	}
6212	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6213	}
6214	return rc;
6215	}
6216
6217	#undef VMXLOCAL_READ_SEG
6218
6219
6220	/**
6221	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6222	* context.
6223	*
6224	* @returns VBox status code.
6225	* @param pVCpu Pointer to the VMCPU.
6226	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6227	* out-of-sync. Make sure to update the required fields
6228	* before using them.
6229	*
6230	* @remarks No-long-jump zone!!!
6231	*/
6232	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6233	{
6234	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6235	{
6236	if (!pVCpu->hm.s.fUsingHyperDR7)
6237	{
6238	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6239	uint32_t u32Val;
6240	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6241	pMixedCtx->dr[7] = u32Val;
6242	}
6243
6244	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6245	}
6246	return VINF_SUCCESS;
6247	}
6248
6249
6250	/**
6251	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6252	*
6253	* @returns VBox status code.
6254	* @param pVCpu Pointer to the VMCPU.
6255	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6256	* out-of-sync. Make sure to update the required fields
6257	* before using them.
6258	*
6259	* @remarks No-long-jump zone!!!
6260	*/
6261	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6262	{
6263	NOREF(pMixedCtx);
6264
6265	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6266	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6267	return VINF_SUCCESS;
6268	}
6269
6270
6271	/**
6272	* Saves the entire guest state from the currently active VMCS into the
6273	* guest-CPU context. This essentially VMREADs all guest-data.
6274	*
6275	* @returns VBox status code.
6276	* @param pVCpu Pointer to the VMCPU.
6277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6278	* out-of-sync. Make sure to update the required fields
6279	* before using them.
6280	*/
6281	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282	{
6283	Assert(pVCpu);
6284	Assert(pMixedCtx);
6285
6286	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6287	return VINF_SUCCESS;
6288
6289	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6290	again on the ring-3 callback path, there is no real need to. */
6291	if (VMMRZCallRing3IsEnabled(pVCpu))
6292	VMMR0LogFlushDisable(pVCpu);
6293	else
6294	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6295	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6296
6297	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6298	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6299
6300	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6301	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6302
6303	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6304	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6305
6306	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6323
6324	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6326
6327	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6328	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6329
6330	if (VMMRZCallRing3IsEnabled(pVCpu))
6331	VMMR0LogFlushEnable(pVCpu);
6332
6333	return rc;
6334	}
6335
6336
6337	/**
6338	* Check per-VM and per-VCPU force flag actions that require us to go back to
6339	* ring-3 for one reason or another.
6340	*
6341	* @returns VBox status code (information status code included).
6342	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6343	* ring-3.
6344	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6345	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6346	* interrupts)
6347	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6348	* all EMTs to be in ring-3.
6349	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6350	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6351	* to the EM loop.
6352	*
6353	* @param pVM Pointer to the VM.
6354	* @param pVCpu Pointer to the VMCPU.
6355	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6356	* out-of-sync. Make sure to update the required fields
6357	* before using them.
6358	*/
6359	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6360	{
6361	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6362
6363	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6364	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6365	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6366	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6367	{
6368	/* We need the control registers now, make sure the guest-CPU context is updated. */
6369	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6370	AssertRCReturn(rc3, rc3);
6371
6372	/* Pending HM CR3 sync. */
6373	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6374	{
6375	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6376	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6377	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6378	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6379	}
6380
6381	/* Pending HM PAE PDPEs. */
6382	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6383	{
6384	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6385	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6386	}
6387
6388	/* Pending PGM C3 sync. */
6389	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6390	{
6391	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6392	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6393	if (rc2 != VINF_SUCCESS)
6394	{
6395	AssertRC(rc2);
6396	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6397	return rc2;
6398	}
6399	}
6400
6401	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6402	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6403	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6404	{
6405	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6406	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6407	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6408	return rc2;
6409	}
6410
6411	/* Pending VM request packets, such as hardware interrupts. */
6412	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6413	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6414	{
6415	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6416	return VINF_EM_PENDING_REQUEST;
6417	}
6418
6419	/* Pending PGM pool flushes. */
6420	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6421	{
6422	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6423	return VINF_PGM_POOL_FLUSH_PENDING;
6424	}
6425
6426	/* Pending DMA requests. */
6427	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6428	{
6429	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6430	return VINF_EM_RAW_TO_R3;
6431	}
6432	}
6433
6434	return VINF_SUCCESS;
6435	}
6436
6437
6438	/**
6439	* Converts any TRPM trap into a pending HM event. This is typically used when
6440	* entering from ring-3 (not longjmp returns).
6441	*
6442	* @param pVCpu Pointer to the VMCPU.
6443	*/
6444	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6445	{
6446	Assert(TRPMHasTrap(pVCpu));
6447	Assert(!pVCpu->hm.s.Event.fPending);
6448
6449	uint8_t uVector;
6450	TRPMEVENT enmTrpmEvent;
6451	RTGCUINT uErrCode;
6452	RTGCUINTPTR GCPtrFaultAddress;
6453	uint8_t cbInstr;
6454
6455	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6456	AssertRC(rc);
6457
6458	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6459	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6460	if (enmTrpmEvent == TRPM_TRAP)
6461	{
6462	switch (uVector)
6463	{
6464	case X86_XCPT_BP:
6465	case X86_XCPT_OF:
6466	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6467	break;
6468
6469	case X86_XCPT_PF:
6470	case X86_XCPT_DF:
6471	case X86_XCPT_TS:
6472	case X86_XCPT_NP:
6473	case X86_XCPT_SS:
6474	case X86_XCPT_GP:
6475	case X86_XCPT_AC:
6476	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6477	/* no break! */
6478	default:
6479	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6480	break;
6481	}
6482	}
6483	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6484	{
6485	if (uVector == X86_XCPT_NMI)
6486	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6487	else
6488	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6489	}
6490	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6491	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6492	else
6493	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6494
6495	rc = TRPMResetTrap(pVCpu);
6496	AssertRC(rc);
6497	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6498	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6499
6500	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6501	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6502	}
6503
6504
6505	/**
6506	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6507	* VT-x to execute any instruction.
6508	*
6509	* @param pvCpu Pointer to the VMCPU.
6510	*/
6511	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6512	{
6513	Assert(pVCpu->hm.s.Event.fPending);
6514
6515	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6516	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6517	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6518	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6519
6520	/* If a trap was already pending, we did something wrong! */
6521	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6522
6523	TRPMEVENT enmTrapType;
6524	switch (uVectorType)
6525	{
6526	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6527	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6528	enmTrapType = TRPM_HARDWARE_INT;
6529	break;
6530
6531	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6532	enmTrapType = TRPM_SOFTWARE_INT;
6533	break;
6534
6535	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6536	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6537	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6538	enmTrapType = TRPM_TRAP;
6539	break;
6540
6541	default:
6542	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6543	enmTrapType = TRPM_32BIT_HACK;
6544	break;
6545	}
6546
6547	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6548
6549	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6550	AssertRC(rc);
6551
6552	if (fErrorCodeValid)
6553	TRPMSetErrorCode(pVCpu, uErrorCode);
6554
6555	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6556	&& uVector == X86_XCPT_PF)
6557	{
6558	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6559	}
6560	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6561	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6562	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6563	{
6564	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6565	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6566	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6567	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6568	}
6569	pVCpu->hm.s.Event.fPending = false;
6570	}
6571
6572
6573	/**
6574	* Does the necessary state syncing before returning to ring-3 for any reason
6575	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6576	*
6577	* @returns VBox status code.
6578	* @param pVM Pointer to the VM.
6579	* @param pVCpu Pointer to the VMCPU.
6580	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6581	* be out-of-sync. Make sure to update the required
6582	* fields before using them.
6583	* @param fSaveGuestState Whether to save the guest state or not.
6584	*
6585	* @remarks No-long-jmp zone!!!
6586	*/
6587	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6588	{
6589	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6590	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6591
6592	RTCPUID idCpu = RTMpCpuId();
6593	Log4Func(("HostCpuId=%u\n", idCpu));
6594
6595	/*
6596	* !!! IMPORTANT !!!
6597	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
6598	*/
6599
6600	/* Save the guest state if necessary. */
6601	if ( fSaveGuestState
6602	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6603	{
6604	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6605	AssertRCReturn(rc, rc);
6606	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6607	}
6608
6609	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6610	if (CPUMIsGuestFPUStateActive(pVCpu))
6611	{
6612	/* We shouldn't reload CR0 without saving it first. */
6613	if (!fSaveGuestState)
6614	{
6615	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6616	AssertRCReturn(rc, rc);
6617	}
6618	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6619	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6620	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6621	}
6622
6623	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6624	#ifdef VBOX_STRICT
6625	if (CPUMIsHyperDebugStateActive(pVCpu))
6626	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6627	#endif
6628	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6629	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6630	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6631	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6632
6633	#if HC_ARCH_BITS == 64
6634	/* Restore host-state bits that VT-x only restores partially. */
6635	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6636	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6637	{
6638	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6639	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6640	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6641	}
6642	#endif
6643
6644	#if HC_ARCH_BITS == 64
6645	/* Restore the host MSRs as we're leaving VT-x context. */
6646	if ( pVM->hm.s.fAllow64BitGuests
6647	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6648	{
6649	/* We shouldn't reload the guest MSRs without saving it first. */
6650	if (!fSaveGuestState)
6651	{
6652	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6653	AssertRCReturn(rc, rc);
6654	}
6655	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6656	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6657	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6658	}
6659	#endif
6660
6661	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6662	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6663	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6664	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6665	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6666	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6667	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6668	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6669
6670	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6671
6672	/** @todo This partially defeats the purpose of having preemption hooks.
6673	* The problem is, deregistering the hooks should be moved to a place that
6674	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6675	* context.
6676	*/
6677	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6678	{
6679	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6680	AssertRCReturn(rc, rc);
6681
6682	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6683	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6684	}
6685	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6686	NOREF(idCpu);
6687
6688	return VINF_SUCCESS;
6689	}
6690
6691
6692	/**
6693	* Leaves the VT-x session.
6694	*
6695	* @returns VBox status code.
6696	* @param pVM Pointer to the VM.
6697	* @param pVCpu Pointer to the VMCPU.
6698	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6699	* out-of-sync. Make sure to update the required fields
6700	* before using them.
6701	*
6702	* @remarks No-long-jmp zone!!!
6703	*/
6704	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6705	{
6706	HM_DISABLE_PREEMPT_IF_NEEDED();
6707	HMVMX_ASSERT_CPU_SAFE();
6708	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6709	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6710
6711	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6712	and done this from the VMXR0ThreadCtxCallback(). */
6713	if (!pVCpu->hm.s.fLeaveDone)
6714	{
6715	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6716	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6717	pVCpu->hm.s.fLeaveDone = true;
6718	}
6719
6720	/*
6721	* !!! IMPORTANT !!!
6722	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
6723	*/
6724
6725	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6726	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6727	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6728	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6729	VMMR0ThreadCtxHooksDeregister(pVCpu);
6730
6731	/* Leave HM context. This takes care of local init (term). */
6732	int rc = HMR0LeaveCpu(pVCpu);
6733
6734	HM_RESTORE_PREEMPT_IF_NEEDED();
6735
6736	return rc;
6737	}
6738
6739
6740	/**
6741	* Does the necessary state syncing before doing a longjmp to ring-3.
6742	*
6743	* @returns VBox status code.
6744	* @param pVM Pointer to the VM.
6745	* @param pVCpu Pointer to the VMCPU.
6746	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6747	* out-of-sync. Make sure to update the required fields
6748	* before using them.
6749	*
6750	* @remarks No-long-jmp zone!!!
6751	*/
6752	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6753	{
6754	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6755	}
6756
6757
6758	/**
6759	* Take necessary actions before going back to ring-3.
6760	*
6761	* An action requires us to go back to ring-3. This function does the necessary
6762	* steps before we can safely return to ring-3. This is not the same as longjmps
6763	* to ring-3, this is voluntary and prepares the guest so it may continue
6764	* executing outside HM (recompiler/IEM).
6765	*
6766	* @returns VBox status code.
6767	* @param pVM Pointer to the VM.
6768	* @param pVCpu Pointer to the VMCPU.
6769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6770	* out-of-sync. Make sure to update the required fields
6771	* before using them.
6772	* @param rcExit The reason for exiting to ring-3. Can be
6773	* VINF_VMM_UNKNOWN_RING3_CALL.
6774	*/
6775	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6776	{
6777	Assert(pVM);
6778	Assert(pVCpu);
6779	Assert(pMixedCtx);
6780	HMVMX_ASSERT_PREEMPT_SAFE();
6781
6782	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6783	{
6784	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6785	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6786	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6787	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6788	}
6789
6790	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6791	VMMRZCallRing3Disable(pVCpu);
6792	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6793
6794	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6795	if (pVCpu->hm.s.Event.fPending)
6796	{
6797	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6798	Assert(!pVCpu->hm.s.Event.fPending);
6799	}
6800
6801	/* Save guest state and restore host state bits. */
6802	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6803	AssertRCReturn(rc, rc);
6804	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6805
6806	/* Sync recompiler state. */
6807	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6808	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6809	\| CPUM_CHANGED_LDTR
6810	\| CPUM_CHANGED_GDTR
6811	\| CPUM_CHANGED_IDTR
6812	\| CPUM_CHANGED_TR
6813	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6814	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6815	if ( pVM->hm.s.fNestedPaging
6816	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6817	{
6818	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6819	}
6820
6821	Assert(!pVCpu->hm.s.fClearTrapFlag);
6822
6823	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6824	if (rcExit != VINF_EM_RAW_INTERRUPT)
6825	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6826
6827	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6828
6829	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6830	VMMRZCallRing3RemoveNotification(pVCpu);
6831	VMMRZCallRing3Enable(pVCpu);
6832
6833	return rc;
6834	}
6835
6836
6837	/**
6838	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6839	* longjump to ring-3 and possibly get preempted.
6840	*
6841	* @returns VBox status code.
6842	* @param pVCpu Pointer to the VMCPU.
6843	* @param enmOperation The operation causing the ring-3 longjump.
6844	* @param pvUser Opaque pointer to the guest-CPU context. The data
6845	* may be out-of-sync. Make sure to update the required
6846	* fields before using them.
6847	*/
6848	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6849	{
6850	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6851	{
6852	/*
6853	* !!! IMPORTANT !!!
6854	* If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6855	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6856	*/
6857	VMMRZCallRing3RemoveNotification(pVCpu);
6858	VMMRZCallRing3Disable(pVCpu);
6859	HM_DISABLE_PREEMPT_IF_NEEDED();
6860
6861	PVM pVM = pVCpu->CTX_SUFF(pVM);
6862	if (CPUMIsGuestFPUStateActive(pVCpu))
6863	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6864
6865	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6866
6867	#if HC_ARCH_BITS == 64
6868	/* Restore host-state bits that VT-x only restores partially. */
6869	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6870	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6871	{
6872	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6873	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6874	}
6875	/* Restore the host MSRs as we're leaving VT-x context. */
6876	if ( pVM->hm.s.fAllow64BitGuests
6877	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6878	{
6879	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6880	}
6881	#endif
6882	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6883	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6884	{
6885	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6886	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6887	}
6888
6889	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6890	VMMR0ThreadCtxHooksDeregister(pVCpu);
6891
6892	HMR0LeaveCpu(pVCpu);
6893	HM_RESTORE_PREEMPT_IF_NEEDED();
6894	return VINF_SUCCESS;
6895	}
6896
6897	Assert(pVCpu);
6898	Assert(pvUser);
6899	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6900	HMVMX_ASSERT_PREEMPT_SAFE();
6901
6902	VMMRZCallRing3Disable(pVCpu);
6903	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6904
6905	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6906	enmOperation));
6907
6908	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6909	AssertRCReturn(rc, rc);
6910
6911	VMMRZCallRing3Enable(pVCpu);
6912	return VINF_SUCCESS;
6913	}
6914
6915
6916	/**
6917	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6918	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6919	*
6920	* @param pVCpu Pointer to the VMCPU.
6921	*/
6922	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6923	{
6924	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6925	{
6926	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6927	{
6928	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6929	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6930	AssertRC(rc);
6931	}
6932	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6933	}
6934
6935
6936	/**
6937	* Evaluates the event to be delivered to the guest and sets it as the pending
6938	* event.
6939	*
6940	* @param pVCpu Pointer to the VMCPU.
6941	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6942	* out-of-sync. Make sure to update the required fields
6943	* before using them.
6944	*/
6945	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6946	{
6947	Assert(!pVCpu->hm.s.Event.fPending);
6948
6949	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6950	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6951	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6952	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6953
6954	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
6955	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6956	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6957	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6958	Assert(!TRPMHasTrap(pVCpu));
6959
6960	/** @todo SMI. SMIs take priority over NMIs. */
6961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6962	{
6963	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6964	if ( !fBlockMovSS
6965	&& !fBlockSti)
6966	{
6967	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6968	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6969	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6970	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6971
6972	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6973	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6974	}
6975	else
6976	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6977	}
6978	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6979	&& !pVCpu->hm.s.fSingleInstruction)
6980	{
6981	/*
6982	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6983	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6984	* evaluated here and not set as pending, solely based on the force-flags.
6985	*/
6986	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6987	AssertRC(rc);
6988	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6989	if ( !fBlockInt
6990	&& !fBlockSti
6991	&& !fBlockMovSS)
6992	{
6993	uint8_t u8Interrupt;
6994	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6995	if (RT_SUCCESS(rc))
6996	{
6997	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6998	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6999	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7000
7001	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7002	}
7003	else
7004	{
7005	/** @todo Does this actually happen? If not turn it into an assertion. */
7006	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7007	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7008	}
7009	}
7010	else
7011	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7012	}
7013	}
7014
7015
7016	/**
7017	* Injects any pending events into the guest if the guest is in a state to
7018	* receive them.
7019	*
7020	* @returns VBox status code (informational status codes included).
7021	* @param pVCpu Pointer to the VMCPU.
7022	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7023	* out-of-sync. Make sure to update the required fields
7024	* before using them.
7025	*/
7026	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7027	{
7028	HMVMX_ASSERT_PREEMPT_SAFE();
7029	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7030
7031	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7032	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7033	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7034	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7035
7036	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7037	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7038	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7039	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7040	Assert(!TRPMHasTrap(pVCpu));
7041
7042	int rc = VINF_SUCCESS;
7043	if (pVCpu->hm.s.Event.fPending)
7044	{
7045	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7046	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7047	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7048	{
7049	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7050	AssertRCReturn(rc, rc);
7051	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7052	Assert(!fBlockInt);
7053	Assert(!fBlockSti);
7054	Assert(!fBlockMovSS);
7055	}
7056	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7057	{
7058	Assert(!fBlockSti);
7059	Assert(!fBlockMovSS);
7060	}
7061	#endif
7062	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7063	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7064	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7065	AssertRCReturn(rc, rc);
7066
7067	/* Update the interruptibility-state as it could have been changed by
7068	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7069	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7070	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7071
7072	#ifdef VBOX_WITH_STATISTICS
7073	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7074	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7075	else
7076	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7077	#endif
7078	}
7079
7080	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7081	if ( !pVCpu->hm.s.fSingleInstruction
7082	&& !DBGFIsStepping(pVCpu))
7083	{
7084	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7085	AssertRCReturn(rc2, rc2);
7086	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7087	{
7088	/*
7089	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7090	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7091	* See Intel spec. 27.3.4 "Saving Non-Register State".
7092	*/
7093	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7094	AssertRCReturn(rc2, rc2);
7095	}
7096	}
7097	else
7098	{
7099	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
7100	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
7101	uIntrState = 0;
7102	}
7103
7104	/*
7105	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7106	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7107	*/
7108	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7109	AssertRC(rc2);
7110
7111	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7112	NOREF(fBlockMovSS); NOREF(fBlockSti);
7113	return rc;
7114	}
7115
7116
7117	/**
7118	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7119	*
7120	* @param pVCpu Pointer to the VMCPU.
7121	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7122	* out-of-sync. Make sure to update the required fields
7123	* before using them.
7124	*/
7125	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7126	{
7127	NOREF(pMixedCtx);
7128	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7129	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7130	}
7131
7132
7133	/**
7134	* Injects a double-fault (#DF) exception into the VM.
7135	*
7136	* @returns VBox status code (informational status code included).
7137	* @param pVCpu Pointer to the VMCPU.
7138	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7139	* out-of-sync. Make sure to update the required fields
7140	* before using them.
7141	*/
7142	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7143	{
7144	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7145	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7146	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7147	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7148	puIntrState);
7149	}
7150
7151
7152	/**
7153	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7154	*
7155	* @param pVCpu Pointer to the VMCPU.
7156	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7157	* out-of-sync. Make sure to update the required fields
7158	* before using them.
7159	*/
7160	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7161	{
7162	NOREF(pMixedCtx);
7163	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7164	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7165	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7166	}
7167
7168
7169	/**
7170	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7171	*
7172	* @param pVCpu Pointer to the VMCPU.
7173	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7174	* out-of-sync. Make sure to update the required fields
7175	* before using them.
7176	* @param cbInstr The value of RIP that is to be pushed on the guest
7177	* stack.
7178	*/
7179	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7180	{
7181	NOREF(pMixedCtx);
7182	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7183	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7184	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7185	}
7186
7187
7188	/**
7189	* Injects a general-protection (#GP) fault into the VM.
7190	*
7191	* @returns VBox status code (informational status code included).
7192	* @param pVCpu Pointer to the VMCPU.
7193	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7194	* out-of-sync. Make sure to update the required fields
7195	* before using them.
7196	* @param u32ErrorCode The error code associated with the #GP.
7197	*/
7198	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7199	uint32_t *puIntrState)
7200	{
7201	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7202	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7203	if (fErrorCodeValid)
7204	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7205	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7206	puIntrState);
7207	}
7208
7209
7210	/**
7211	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7212	*
7213	* @param pVCpu Pointer to the VMCPU.
7214	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7215	* out-of-sync. Make sure to update the required fields
7216	* before using them.
7217	* @param uVector The software interrupt vector number.
7218	* @param cbInstr The value of RIP that is to be pushed on the guest
7219	* stack.
7220	*/
7221	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7222	{
7223	NOREF(pMixedCtx);
7224	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7225	if ( uVector == X86_XCPT_BP
7226	\|\| uVector == X86_XCPT_OF)
7227	{
7228	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7229	}
7230	else
7231	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7232	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7233	}
7234
7235
7236	/**
7237	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7238	* stack.
7239	*
7240	* @returns VBox status code (information status code included).
7241	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7242	* @param pVM Pointer to the VM.
7243	* @param pMixedCtx Pointer to the guest-CPU context.
7244	* @param uValue The value to push to the guest stack.
7245	*/
7246	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7247	{
7248	/*
7249	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7250	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7251	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7252	*/
7253	if (pMixedCtx->sp == 1)
7254	return VINF_EM_RESET;
7255	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7256	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7257	AssertRCReturn(rc, rc);
7258	return rc;
7259	}
7260
7261
7262	/**
7263	* Injects an event into the guest upon VM-entry by updating the relevant fields
7264	* in the VM-entry area in the VMCS.
7265	*
7266	* @returns VBox status code (informational error codes included).
7267	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7268	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7269	*
7270	* @param pVCpu Pointer to the VMCPU.
7271	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7272	* be out-of-sync. Make sure to update the required
7273	* fields before using them.
7274	* @param u64IntInfo The VM-entry interruption-information field.
7275	* @param cbInstr The VM-entry instruction length in bytes (for
7276	* software interrupts, exceptions and privileged
7277	* software exceptions).
7278	* @param u32ErrCode The VM-entry exception error code.
7279	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7280	* @param puIntrState Pointer to the current guest interruptibility-state.
7281	* This interruptibility-state will be updated if
7282	* necessary. This cannot not be NULL.
7283	*
7284	* @remarks Requires CR0!
7285	* @remarks No-long-jump zone!!!
7286	*/
7287	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7288	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7289	{
7290	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7291	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7292	Assert(puIntrState);
7293	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7294
7295	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7296	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7297
7298	#ifdef VBOX_STRICT
7299	/* Validate the error-code-valid bit for hardware exceptions. */
7300	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7301	{
7302	switch (uVector)
7303	{
7304	case X86_XCPT_PF:
7305	case X86_XCPT_DF:
7306	case X86_XCPT_TS:
7307	case X86_XCPT_NP:
7308	case X86_XCPT_SS:
7309	case X86_XCPT_GP:
7310	case X86_XCPT_AC:
7311	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7312	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7313	/* fallthru */
7314	default:
7315	break;
7316	}
7317	}
7318	#endif
7319
7320	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7321	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7322	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7323
7324	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7325
7326	/* We require CR0 to check if the guest is in real-mode. */
7327	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7328	AssertRCReturn(rc, rc);
7329
7330	/*
7331	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7332	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7333	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7334	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7335	*/
7336	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7337	{
7338	PVM pVM = pVCpu->CTX_SUFF(pVM);
7339	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7340	{
7341	Assert(PDMVmmDevHeapIsEnabled(pVM));
7342	Assert(pVM->hm.s.vmx.pRealModeTSS);
7343
7344	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7345	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7346	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7347	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7348	AssertRCReturn(rc, rc);
7349	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7350
7351	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7352	const size_t cbIdtEntry = sizeof(X86IDTR16);
7353	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7354	{
7355	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7356	if (uVector == X86_XCPT_DF)
7357	return VINF_EM_RESET;
7358	else if (uVector == X86_XCPT_GP)
7359	{
7360	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7361	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7362	}
7363
7364	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7365	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7366	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7367	}
7368
7369	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7370	uint16_t uGuestIp = pMixedCtx->ip;
7371	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7372	{
7373	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7374	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7375	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7376	}
7377	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7378	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7379
7380	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7381	X86IDTR16 IdtEntry;
7382	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7383	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7384	AssertRCReturn(rc, rc);
7385
7386	/* Construct the stack frame for the interrupt/exception handler. */
7387	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7388	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7389	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7390	AssertRCReturn(rc, rc);
7391
7392	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7393	if (rc == VINF_SUCCESS)
7394	{
7395	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7396	pMixedCtx->rip = IdtEntry.offSel;
7397	pMixedCtx->cs.Sel = IdtEntry.uSel;
7398	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7399	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7400	&& uVector == X86_XCPT_PF)
7401	{
7402	pMixedCtx->cr2 = GCPtrFaultAddress;
7403	}
7404
7405	/* If any other guest-state bits are changed here, make sure to update
7406	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7407	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7408	\| HM_CHANGED_GUEST_RIP
7409	\| HM_CHANGED_GUEST_RFLAGS
7410	\| HM_CHANGED_GUEST_RSP);
7411
7412	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7413	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7414	{
7415	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7416	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7417	Log4(("Clearing inhibition due to STI.\n"));
7418	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7419	}
7420	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7421
7422	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7423	it, if we are returning to ring-3 before executing guest code. */
7424	pVCpu->hm.s.Event.fPending = false;
7425	}
7426	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7427	return rc;
7428	}
7429	else
7430	{
7431	/*
7432	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7433	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7434	*/
7435	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7436	}
7437	}
7438
7439	/* Validate. */
7440	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7441	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7442	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7443
7444	/* Inject. */
7445	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7446	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7447	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7448	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7449
7450	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7451	&& uVector == X86_XCPT_PF)
7452	{
7453	pMixedCtx->cr2 = GCPtrFaultAddress;
7454	}
7455
7456	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7457	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7458
7459	AssertRCReturn(rc, rc);
7460	return rc;
7461	}
7462
7463
7464	/**
7465	* Clears the interrupt-window exiting control in the VMCS and if necessary
7466	* clears the current event in the VMCS as well.
7467	*
7468	* @returns VBox status code.
7469	* @param pVCpu Pointer to the VMCPU.
7470	*
7471	* @remarks Use this function only to clear events that have not yet been
7472	* delivered to the guest but are injected in the VMCS!
7473	* @remarks No-long-jump zone!!!
7474	*/
7475	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7476	{
7477	int rc;
7478	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7479
7480	/* Clear interrupt-window exiting control. */
7481	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7482	{
7483	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7484	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7485	AssertRC(rc);
7486	}
7487
7488	if (!pVCpu->hm.s.Event.fPending)
7489	return;
7490
7491	#ifdef VBOX_STRICT
7492	uint32_t u32EntryInfo;
7493	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7494	AssertRC(rc);
7495	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7496	#endif
7497
7498	/* Clear the entry-interruption field (including the valid bit). */
7499	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7500	AssertRC(rc);
7501
7502	/* Clear the pending debug exception field. */
7503	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7504	AssertRC(rc);
7505	}
7506
7507
7508	/**
7509	* Enters the VT-x session.
7510	*
7511	* @returns VBox status code.
7512	* @param pVM Pointer to the VM.
7513	* @param pVCpu Pointer to the VMCPU.
7514	* @param pCpu Pointer to the CPU info struct.
7515	*/
7516	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7517	{
7518	AssertPtr(pVM);
7519	AssertPtr(pVCpu);
7520	Assert(pVM->hm.s.vmx.fSupported);
7521	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7522	NOREF(pCpu); NOREF(pVM);
7523
7524	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7525	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7526
7527	#ifdef VBOX_STRICT
7528	/* Make sure we're in VMX root mode. */
7529	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7530	if (!(u32HostCR4 & X86_CR4_VMXE))
7531	{
7532	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7533	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7534	}
7535	#endif
7536
7537	/*
7538	* Load the VCPU's VMCS as the current (and active) one.
7539	*/
7540	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7541	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7542	if (RT_FAILURE(rc))
7543	return rc;
7544
7545	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7546	pVCpu->hm.s.fLeaveDone = false;
7547	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7548
7549	return VINF_SUCCESS;
7550	}
7551
7552
7553	/**
7554	* The thread-context callback (only on platforms which support it).
7555	*
7556	* @param enmEvent The thread-context event.
7557	* @param pVCpu Pointer to the VMCPU.
7558	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7559	* @thread EMT(pVCpu)
7560	*/
7561	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7562	{
7563	NOREF(fGlobalInit);
7564
7565	switch (enmEvent)
7566	{
7567	case RTTHREADCTXEVENT_PREEMPTING:
7568	{
7569	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7570	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7571	VMCPU_ASSERT_EMT(pVCpu);
7572
7573	PVM pVM = pVCpu->CTX_SUFF(pVM);
7574	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7575
7576	/* No longjmps (logger flushes, locks) in this fragile context. */
7577	VMMRZCallRing3Disable(pVCpu);
7578	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7579
7580	/*
7581	* Restore host-state (FPU, debug etc.)
7582	*/
7583	if (!pVCpu->hm.s.fLeaveDone)
7584	{
7585	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7586	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7587	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7588	pVCpu->hm.s.fLeaveDone = true;
7589	}
7590
7591	/* Leave HM context, takes care of local init (term). */
7592	int rc = HMR0LeaveCpu(pVCpu);
7593	AssertRC(rc); NOREF(rc);
7594
7595	/* Restore longjmp state. */
7596	VMMRZCallRing3Enable(pVCpu);
7597	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7598	break;
7599	}
7600
7601	case RTTHREADCTXEVENT_RESUMED:
7602	{
7603	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7604	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7605	VMCPU_ASSERT_EMT(pVCpu);
7606
7607	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7608	VMMRZCallRing3Disable(pVCpu);
7609	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7610
7611	/* Initialize the bare minimum state required for HM. This takes care of
7612	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7613	int rc = HMR0EnterCpu(pVCpu);
7614	AssertRC(rc);
7615	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7616
7617	/* Load the active VMCS as the current one. */
7618	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7619	{
7620	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7621	AssertRC(rc); NOREF(rc);
7622	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7623	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7624	}
7625	pVCpu->hm.s.fLeaveDone = false;
7626
7627	/* Restore longjmp state. */
7628	VMMRZCallRing3Enable(pVCpu);
7629	break;
7630	}
7631
7632	default:
7633	break;
7634	}
7635	}
7636
7637
7638	/**
7639	* Saves the host state in the VMCS host-state.
7640	* Sets up the VM-exit MSR-load area.
7641	*
7642	* The CPU state will be loaded from these fields on every successful VM-exit.
7643	*
7644	* @returns VBox status code.
7645	* @param pVM Pointer to the VM.
7646	* @param pVCpu Pointer to the VMCPU.
7647	*
7648	* @remarks No-long-jump zone!!!
7649	*/
7650	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7651	{
7652	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7653
7654	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7655	return VINF_SUCCESS;
7656
7657	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7658	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7659
7660	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7661	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7662
7663	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7664	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7665
7666	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7667	return rc;
7668	}
7669
7670
7671	/**
7672	* Saves the host state in the VMCS host-state.
7673	*
7674	* @returns VBox status code.
7675	* @param pVM Pointer to the VM.
7676	* @param pVCpu Pointer to the VMCPU.
7677	*
7678	* @remarks No-long-jump zone!!!
7679	*/
7680	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7681	{
7682	AssertPtr(pVM);
7683	AssertPtr(pVCpu);
7684
7685	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7686
7687	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7688	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7689	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7690	return hmR0VmxSaveHostState(pVM, pVCpu);
7691	}
7692
7693
7694	/**
7695	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7696	* loaded from these fields on every successful VM-entry.
7697	*
7698	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7699	* Sets up the VM-entry controls.
7700	* Sets up the appropriate VMX non-root function to execute guest code based on
7701	* the guest CPU mode.
7702	*
7703	* @returns VBox status code.
7704	* @param pVM Pointer to the VM.
7705	* @param pVCpu Pointer to the VMCPU.
7706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7707	* out-of-sync. Make sure to update the required fields
7708	* before using them.
7709	*
7710	* @remarks No-long-jump zone!!!
7711	*/
7712	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7713	{
7714	AssertPtr(pVM);
7715	AssertPtr(pVCpu);
7716	AssertPtr(pMixedCtx);
7717	HMVMX_ASSERT_PREEMPT_SAFE();
7718
7719	#ifdef LOG_ENABLED
7720	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7721	* probably not initialized yet? Anyway this will do for now.
7722	*
7723	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7724	* interface and disable ring-3 calls when thread-context hooks are not
7725	* available. */
7726	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7727	VMMR0LogFlushDisable(pVCpu);
7728	#endif
7729
7730	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7731
7732	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7733
7734	/* Determine real-on-v86 mode. */
7735	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7736	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7737	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7738	{
7739	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7740	}
7741
7742	/*
7743	* Load the guest-state into the VMCS.
7744	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7745	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7746	*/
7747	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7748	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7749
7750	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7751	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7752	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7753
7754	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7755	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7756	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7757
7758	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7760
7761	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7762	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7763
7764	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7765	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7766	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7767
7768	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7769	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7770
7771	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7772	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7773
7774	/*
7775	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7776	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7777	*/
7778	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7779	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7780
7781	/* Clear any unused and reserved bits. */
7782	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7783
7784	#ifdef LOG_ENABLED
7785	/* Only reenable log-flushing if the caller has it enabled. */
7786	if (!fCallerDisabledLogFlush)
7787	VMMR0LogFlushEnable(pVCpu);
7788	#endif
7789
7790	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7791	return rc;
7792	}
7793
7794
7795	/**
7796	* Loads the state shared between the host and guest into the VMCS.
7797	*
7798	* @param pVM Pointer to the VM.
7799	* @param pVCpu Pointer to the VMCPU.
7800	* @param pCtx Pointer to the guest-CPU context.
7801	*
7802	* @remarks No-long-jump zone!!!
7803	*/
7804	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7805	{
7806	NOREF(pVM);
7807
7808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7809	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7810
7811	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7812	{
7813	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7814	AssertRC(rc);
7815	}
7816
7817	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7818	{
7819	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7820	AssertRC(rc);
7821
7822	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7823	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7824	{
7825	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7826	AssertRC(rc);
7827	}
7828	}
7829
7830	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7831	{
7832	#if HC_ARCH_BITS == 64
7833	if (pVM->hm.s.fAllow64BitGuests)
7834	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7835	#endif
7836	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7837	}
7838
7839	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7840	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7841	}
7842
7843
7844	/**
7845	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7846	*
7847	* @param pVM Pointer to the VM.
7848	* @param pVCpu Pointer to the VMCPU.
7849	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7850	* out-of-sync. Make sure to update the required fields
7851	* before using them.
7852	*/
7853	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7854	{
7855	HMVMX_ASSERT_PREEMPT_SAFE();
7856
7857	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7858	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7859	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7860	#endif
7861
7862	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7863	{
7864	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7865	AssertRC(rc);
7866	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7867	}
7868	else if (HMCPU_CF_VALUE(pVCpu))
7869	{
7870	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7871	AssertRC(rc);
7872	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7873	}
7874
7875	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7876	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7877	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7878	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7879
7880	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7881	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7882	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7883	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7884	#endif
7885	}
7886
7887
7888	/**
7889	* Does the preparations before executing guest code in VT-x.
7890	*
7891	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7892	* recompiler. We must be cautious what we do here regarding committing
7893	* guest-state information into the VMCS assuming we assuredly execute the
7894	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7895	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7896	* so that the recompiler can (and should) use them when it resumes guest
7897	* execution. Otherwise such operations must be done when we can no longer
7898	* exit to ring-3.
7899	*
7900	* @returns Strict VBox status code.
7901	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7902	* have been disabled.
7903	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7904	* double-fault into the guest.
7905	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7906	*
7907	* @param pVM Pointer to the VM.
7908	* @param pVCpu Pointer to the VMCPU.
7909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7910	* out-of-sync. Make sure to update the required fields
7911	* before using them.
7912	* @param pVmxTransient Pointer to the VMX transient structure.
7913	*/
7914	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7915	{
7916	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7917
7918	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7919	PGMRZDynMapFlushAutoSet(pVCpu);
7920	#endif
7921
7922	/* Check force flag actions that might require us to go back to ring-3. */
7923	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7924	if (rc != VINF_SUCCESS)
7925	return rc;
7926
7927	#ifndef IEM_VERIFICATION_MODE_FULL
7928	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7929	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7930	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7931	{
7932	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7933	RTGCPHYS GCPhysApicBase;
7934	GCPhysApicBase = pMixedCtx->msrApicBase;
7935	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7936
7937	/* Unalias any existing mapping. */
7938	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7939	AssertRCReturn(rc, rc);
7940
7941	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7942	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7943	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7944	AssertRCReturn(rc, rc);
7945
7946	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7947	}
7948	#endif /* !IEM_VERIFICATION_MODE_FULL */
7949
7950	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7951	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7952
7953	/*
7954	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7955	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7956	*/
7957	if (TRPMHasTrap(pVCpu))
7958	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7959	else if (!pVCpu->hm.s.Event.fPending)
7960	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7961
7962	/*
7963	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7964	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7965	*/
7966	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7967	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7968	{
7969	Assert(rc == VINF_EM_RESET);
7970	return rc;
7971	}
7972
7973	/*
7974	* No longjmps to ring-3 from this point on!!!
7975	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7976	* This also disables flushing of the R0-logger instance (if any).
7977	*/
7978	VMMRZCallRing3Disable(pVCpu);
7979
7980	/*
7981	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7982	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7983	*
7984	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7985	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7986	*
7987	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7988	* executing guest code.
7989	*/
7990	pVmxTransient->uEflags = ASMIntDisableFlags();
7991	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7992	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7993	{
7994	hmR0VmxClearEventVmcs(pVCpu);
7995	ASMSetFlags(pVmxTransient->uEflags);
7996	VMMRZCallRing3Enable(pVCpu);
7997	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7998	return VINF_EM_RAW_TO_R3;
7999	}
8000	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8001	{
8002	hmR0VmxClearEventVmcs(pVCpu);
8003	ASMSetFlags(pVmxTransient->uEflags);
8004	VMMRZCallRing3Enable(pVCpu);
8005	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8006	return VINF_EM_RAW_INTERRUPT;
8007	}
8008
8009	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8010	pVCpu->hm.s.Event.fPending = false;
8011
8012	return VINF_SUCCESS;
8013	}
8014
8015
8016	/**
8017	* Prepares to run guest code in VT-x and we've committed to doing so. This
8018	* means there is no backing out to ring-3 or anywhere else at this
8019	* point.
8020	*
8021	* @param pVM Pointer to the VM.
8022	* @param pVCpu Pointer to the VMCPU.
8023	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8024	* out-of-sync. Make sure to update the required fields
8025	* before using them.
8026	* @param pVmxTransient Pointer to the VMX transient structure.
8027	*
8028	* @remarks Called with preemption disabled.
8029	* @remarks No-long-jump zone!!!
8030	*/
8031	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8032	{
8033	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8034	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8035	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8036
8037	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8038	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8039
8040	/*
8041	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8042	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8043	* Reload only the necessary state, the assertion will catch if other parts of the code
8044	* change.
8045	*/
8046	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8047	{
8048	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8049	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8050	}
8051
8052	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8053	if (!CPUMIsGuestFPUStateActive(pVCpu))
8054	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8055	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8056	#endif
8057
8058	if ( pVCpu->hm.s.fUseGuestFpu
8059	&& !CPUMIsGuestFPUStateActive(pVCpu))
8060	{
8061	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8062	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8063	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8064	}
8065
8066	/*
8067	* The host MSR values the very first time around won't be updated, so we need to
8068	* fill those values in. Subsequently, it's updated as part of the host state.
8069	*/
8070	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8071	&& pVCpu->hm.s.vmx.cMsrs > 0)
8072	{
8073	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8074	}
8075
8076	/*
8077	* Load the host state bits as we may've been preempted (only happens when
8078	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8079	*/
8080	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8081	{
8082	/* This ASSUMES that pfnStartVM has been set up already. */
8083	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8084	AssertRC(rc);
8085	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8086	}
8087	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8088
8089	/*
8090	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8091	*/
8092	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8093	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8094	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8095
8096	/* Store status of the shared guest-host state at the time of VM-entry. */
8097	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8098	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8099	{
8100	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8101	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8102	}
8103	else
8104	#endif
8105	{
8106	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8107	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8108	}
8109	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8110
8111	/*
8112	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8113	*/
8114	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8115	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8116
8117	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8118	RTCPUID idCurrentCpu = pCpu->idCpu;
8119	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8120	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8121	{
8122	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8123	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8124	}
8125
8126	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8127	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8128	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8129	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8130
8131	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8132
8133	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8134	to start executing. */
8135
8136	/*
8137	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8138	*/
8139	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8140	{
8141	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8142	{
8143	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8144	AssertRC(rc2);
8145	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8146	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */);
8147	}
8148	else
8149	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8150	}
8151	#ifdef VBOX_STRICT
8152	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8153	#endif
8154	}
8155
8156
8157	/**
8158	* Performs some essential restoration of state after running guest code in
8159	* VT-x.
8160	*
8161	* @param pVM Pointer to the VM.
8162	* @param pVCpu Pointer to the VMCPU.
8163	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8164	* out-of-sync. Make sure to update the required fields
8165	* before using them.
8166	* @param pVmxTransient Pointer to the VMX transient structure.
8167	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8168	*
8169	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8170	*
8171	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8172	* unconditionally when it is safe to do so.
8173	*/
8174	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8175	{
8176	NOREF(pVM);
8177
8178	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8179
8180	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8181	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8182	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8183	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8184	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8185
8186	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8187	{
8188	/** @todo Find a way to fix hardcoding a guestimate. */
8189	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8190	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8191	}
8192
8193	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8194	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8195	Assert(!(ASMGetFlags() & X86_EFL_IF));
8196	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8197
8198	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8199	if (CPUMIsGuestFPUStateActive(pVCpu))
8200	{
8201	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8202	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8203	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8204	}
8205	#endif
8206
8207	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8208	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8209	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8210	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8211
8212	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8213	uint32_t uExitReason;
8214	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8215	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8216	AssertRC(rc);
8217	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8218	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8219
8220	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8221	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8222	{
8223	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8224	pVmxTransient->fVMEntryFailed));
8225	return;
8226	}
8227
8228	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8229	{
8230	/* Update the guest interruptibility-state from the VMCS. */
8231	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8232	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8233	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8234	AssertRC(rc);
8235	#endif
8236	/*
8237	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8238	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8239	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8240	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8241	*/
8242	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8243	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8244	{
8245	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8246	AssertRC(rc);
8247	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8248	}
8249	}
8250	}
8251
8252
8253
8254	/**
8255	* Runs the guest code using VT-x the normal way.
8256	*
8257	* @returns VBox status code.
8258	* @param pVM Pointer to the VM.
8259	* @param pVCpu Pointer to the VMCPU.
8260	* @param pCtx Pointer to the guest-CPU context.
8261	*
8262	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8263	*/
8264	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8265	{
8266	VMXTRANSIENT VmxTransient;
8267	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8268	int rc = VERR_INTERNAL_ERROR_5;
8269	uint32_t cLoops = 0;
8270
8271	for (;; cLoops++)
8272	{
8273	Assert(!HMR0SuspendPending());
8274	HMVMX_ASSERT_CPU_SAFE();
8275
8276	/* Preparatory work for running guest code, this may force us to return
8277	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8278	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8279	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8280	if (rc != VINF_SUCCESS)
8281	break;
8282
8283	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8284	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8285	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8286
8287	/* Restore any residual host-state and save any bits shared between host
8288	and guest into the guest-CPU state. Re-enables interrupts! */
8289	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8290
8291	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8292	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8293	{
8294	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8295	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8296	return rc;
8297	}
8298
8299	/* Handle the VM-exit. */
8300	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8302	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8303	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8304	HMVMX_START_EXIT_DISPATCH_PROF();
8305	#ifdef HMVMX_USE_FUNCTION_TABLE
8306	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8307	#else
8308	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8309	#endif
8310	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8311	if (rc != VINF_SUCCESS)
8312	break;
8313	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8314	{
8315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8316	rc = VINF_EM_RAW_INTERRUPT;
8317	break;
8318	}
8319	}
8320
8321	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8322	return rc;
8323	}
8324
8325
8326	/**
8327	* Single steps guest code using VT-x.
8328	*
8329	* @returns VBox status code.
8330	* @param pVM Pointer to the VM.
8331	* @param pVCpu Pointer to the VMCPU.
8332	* @param pCtx Pointer to the guest-CPU context.
8333	*
8334	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8335	*/
8336	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8337	{
8338	VMXTRANSIENT VmxTransient;
8339	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8340	int rc = VERR_INTERNAL_ERROR_5;
8341	uint32_t cLoops = 0;
8342	uint16_t uCsStart = pCtx->cs.Sel;
8343	uint64_t uRipStart = pCtx->rip;
8344
8345	for (;; cLoops++)
8346	{
8347	Assert(!HMR0SuspendPending());
8348	HMVMX_ASSERT_CPU_SAFE();
8349
8350	/* Preparatory work for running guest code, this may force us to return
8351	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8352	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8353	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8354	if (rc != VINF_SUCCESS)
8355	break;
8356
8357	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8358	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8359	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8360
8361	/* Restore any residual host-state and save any bits shared between host
8362	and guest into the guest-CPU state. Re-enables interrupts! */
8363	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8364
8365	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8366	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8367	{
8368	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8369	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8370	return rc;
8371	}
8372
8373	/* Handle the VM-exit. */
8374	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8376	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8377	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8378	HMVMX_START_EXIT_DISPATCH_PROF();
8379	#ifdef HMVMX_USE_FUNCTION_TABLE
8380	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8381	#else
8382	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8383	#endif
8384	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8385	if (rc != VINF_SUCCESS)
8386	break;
8387	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8388	{
8389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8390	rc = VINF_EM_RAW_INTERRUPT;
8391	break;
8392	}
8393
8394	/*
8395	* Did the RIP change, if so, consider it a single step.
8396	* Otherwise, make sure one of the TFs gets set.
8397	*/
8398	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8399	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8400	AssertRCReturn(rc2, rc2);
8401	if ( pCtx->rip != uRipStart
8402	\|\| pCtx->cs.Sel != uCsStart)
8403	{
8404	rc = VINF_EM_DBG_STEPPED;
8405	break;
8406	}
8407	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8408	}
8409
8410	/*
8411	* Clear the X86_EFL_TF if necessary.
8412	*/
8413	if (pVCpu->hm.s.fClearTrapFlag)
8414	{
8415	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8416	AssertRCReturn(rc2, rc2);
8417	pVCpu->hm.s.fClearTrapFlag = false;
8418	pCtx->eflags.Bits.u1TF = 0;
8419	}
8420	/** @todo there seems to be issues with the resume flag when the monitor trap
8421	* flag is pending without being used. Seen early in bios init when
8422	* accessing APIC page in prot mode. */
8423
8424	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8425	return rc;
8426	}
8427
8428
8429	/**
8430	* Runs the guest code using VT-x.
8431	*
8432	* @returns VBox status code.
8433	* @param pVM Pointer to the VM.
8434	* @param pVCpu Pointer to the VMCPU.
8435	* @param pCtx Pointer to the guest-CPU context.
8436	*/
8437	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8438	{
8439	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8440	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8441	HMVMX_ASSERT_PREEMPT_SAFE();
8442
8443	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8444
8445	int rc;
8446	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8447	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8448	else
8449	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8450
8451	if (rc == VERR_EM_INTERPRETER)
8452	rc = VINF_EM_RAW_EMULATE_INSTR;
8453	else if (rc == VINF_EM_RESET)
8454	rc = VINF_EM_TRIPLE_FAULT;
8455
8456	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8457	if (RT_FAILURE(rc2))
8458	{
8459	pVCpu->hm.s.u32HMError = rc;
8460	rc = rc2;
8461	}
8462	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8463	return rc;
8464	}
8465
8466
8467	#ifndef HMVMX_USE_FUNCTION_TABLE
8468	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8469	{
8470	#ifdef DEBUG_ramshankar
8471	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8472	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8473	#endif
8474	int rc;
8475	switch (rcReason)
8476	{
8477	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8478	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8479	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8480	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8481	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8482	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8483	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8484	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8485	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8486	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8487	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8488	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8508	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8509	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8510
8511	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8512	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8513	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8514	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8515	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8516	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8517	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8518	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8519	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8520
8521	case VMX_EXIT_VMCALL:
8522	case VMX_EXIT_VMCLEAR:
8523	case VMX_EXIT_VMLAUNCH:
8524	case VMX_EXIT_VMPTRLD:
8525	case VMX_EXIT_VMPTRST:
8526	case VMX_EXIT_VMREAD:
8527	case VMX_EXIT_VMRESUME:
8528	case VMX_EXIT_VMWRITE:
8529	case VMX_EXIT_VMXOFF:
8530	case VMX_EXIT_VMXON:
8531	case VMX_EXIT_INVEPT:
8532	case VMX_EXIT_INVVPID:
8533	case VMX_EXIT_VMFUNC:
8534	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8535	break;
8536	default:
8537	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8538	break;
8539	}
8540	return rc;
8541	}
8542	#endif
8543
8544	#ifdef DEBUG
8545	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8546	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8547	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8548
8549	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8550	do \
8551	{ \
8552	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8553	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8554	} while (0)
8555
8556	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8557	do { \
8558	AssertPtr(pVCpu); \
8559	AssertPtr(pMixedCtx); \
8560	AssertPtr(pVmxTransient); \
8561	Assert(pVmxTransient->fVMEntryFailed == false); \
8562	Assert(ASMIntAreEnabled()); \
8563	HMVMX_ASSERT_PREEMPT_SAFE(); \
8564	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8565	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8566	HMVMX_ASSERT_PREEMPT_SAFE(); \
8567	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8568	HMVMX_ASSERT_PREEMPT_CPUID(); \
8569	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8570	} while (0)
8571
8572	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8573	do { \
8574	Log4Func(("\n")); \
8575	} while (0)
8576	#else /* Release builds */
8577	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8578	do { \
8579	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8580	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8581	} while (0)
8582	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8583	#endif
8584
8585
8586	/**
8587	* Advances the guest RIP after reading it from the VMCS.
8588	*
8589	* @returns VBox status code.
8590	* @param pVCpu Pointer to the VMCPU.
8591	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8592	* out-of-sync. Make sure to update the required fields
8593	* before using them.
8594	* @param pVmxTransient Pointer to the VMX transient structure.
8595	*
8596	* @remarks No-long-jump zone!!!
8597	*/
8598	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8599	{
8600	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8601	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8602	AssertRCReturn(rc, rc);
8603
8604	pMixedCtx->rip += pVmxTransient->cbInstr;
8605	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8606	return rc;
8607	}
8608
8609
8610	/**
8611	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8612	* and update error record fields accordingly.
8613	*
8614	* @return VMX_IGS_* return codes.
8615	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8616	* wrong with the guest state.
8617	*
8618	* @param pVM Pointer to the VM.
8619	* @param pVCpu Pointer to the VMCPU.
8620	* @param pCtx Pointer to the guest-CPU state.
8621	*/
8622	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8623	{
8624	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8625	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8626	uError = (err); \
8627	break; \
8628	} else do { } while (0)
8629
8630	int rc;
8631	uint32_t uError = VMX_IGS_ERROR;
8632	uint32_t u32Val;
8633	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8634
8635	do
8636	{
8637	/*
8638	* CR0.
8639	*/
8640	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8641	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8642	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8643	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8644	if (fUnrestrictedGuest)
8645	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8646
8647	uint32_t u32GuestCR0;
8648	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8649	AssertRCBreak(rc);
8650	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8651	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8652	if ( !fUnrestrictedGuest
8653	&& (u32GuestCR0 & X86_CR0_PG)
8654	&& !(u32GuestCR0 & X86_CR0_PE))
8655	{
8656	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8657	}
8658
8659	/*
8660	* CR4.
8661	*/
8662	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8663	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8664
8665	uint32_t u32GuestCR4;
8666	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8667	AssertRCBreak(rc);
8668	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8669	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8670
8671	/*
8672	* IA32_DEBUGCTL MSR.
8673	*/
8674	uint64_t u64Val;
8675	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8676	AssertRCBreak(rc);
8677	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8678	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8679	{
8680	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8681	}
8682	uint64_t u64DebugCtlMsr = u64Val;
8683
8684	#ifdef VBOX_STRICT
8685	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8686	AssertRCBreak(rc);
8687	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8688	#endif
8689	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8690
8691	/*
8692	* RIP and RFLAGS.
8693	*/
8694	uint32_t u32Eflags;
8695	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8696	if (HMVMX_IS_64BIT_HOST_MODE())
8697	{
8698	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8699	AssertRCBreak(rc);
8700	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8701	if ( !fLongModeGuest
8702	\|\| !pCtx->cs.Attr.n.u1Long)
8703	{
8704	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8705	}
8706	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8707	* must be identical if the "IA32e mode guest" VM-entry control is 1
8708	* and CS.L is 1. No check applies if the CPU supports 64
8709	* linear-address bits. */
8710
8711	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8712	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8713	AssertRCBreak(rc);
8714	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8715	VMX_IGS_RFLAGS_RESERVED);
8716	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8717	u32Eflags = u64Val;
8718	}
8719	else
8720	#endif
8721	{
8722	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8723	AssertRCBreak(rc);
8724	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8725	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8726	}
8727
8728	if ( fLongModeGuest
8729	\|\| ( fUnrestrictedGuest
8730	&& !(u32GuestCR0 & X86_CR0_PE)))
8731	{
8732	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8733	}
8734
8735	uint32_t u32EntryInfo;
8736	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8737	AssertRCBreak(rc);
8738	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8739	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8740	{
8741	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8742	}
8743
8744	/*
8745	* 64-bit checks.
8746	*/
8747	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8748	if (HMVMX_IS_64BIT_HOST_MODE())
8749	{
8750	if ( fLongModeGuest
8751	&& !fUnrestrictedGuest)
8752	{
8753	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8754	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8755	}
8756
8757	if ( !fLongModeGuest
8758	&& (u32GuestCR4 & X86_CR4_PCIDE))
8759	{
8760	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8761	}
8762
8763	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8764	* 51:32 beyond the processor's physical-address width are 0. */
8765
8766	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8767	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8768	{
8769	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8770	}
8771
8772	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8773	AssertRCBreak(rc);
8774	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8775
8776	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8777	AssertRCBreak(rc);
8778	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8779	}
8780	#endif
8781
8782	/*
8783	* PERF_GLOBAL MSR.
8784	*/
8785	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8786	{
8787	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8788	AssertRCBreak(rc);
8789	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8790	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8791	}
8792
8793	/*
8794	* PAT MSR.
8795	*/
8796	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8797	{
8798	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8799	AssertRCBreak(rc);
8800	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8801	for (unsigned i = 0; i < 8; i++)
8802	{
8803	uint8_t u8Val = (u64Val & 0x7);
8804	if ( u8Val != 0 /* UC */
8805	\|\| u8Val != 1 /* WC */
8806	\|\| u8Val != 4 /* WT */
8807	\|\| u8Val != 5 /* WP */
8808	\|\| u8Val != 6 /* WB */
8809	\|\| u8Val != 7 /* UC- */)
8810	{
8811	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8812	}
8813	u64Val >>= 3;
8814	}
8815	}
8816
8817	/*
8818	* EFER MSR.
8819	*/
8820	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8821	{
8822	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8823	AssertRCBreak(rc);
8824	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8825	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8826	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8827	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8828	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8829	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8830	}
8831
8832	/*
8833	* Segment registers.
8834	*/
8835	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8836	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8837	if (!(u32Eflags & X86_EFL_VM))
8838	{
8839	/* CS */
8840	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8841	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8842	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8843	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8844	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8845	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8846	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8847	/* CS cannot be loaded with NULL in protected mode. */
8848	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8849	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8850	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8851	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8852	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8853	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8854	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8855	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8856	else
8857	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8858
8859	/* SS */
8860	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8861	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8862	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8863	if ( !(pCtx->cr0 & X86_CR0_PE)
8864	\|\| pCtx->cs.Attr.n.u4Type == 3)
8865	{
8866	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8867	}
8868	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8869	{
8870	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8871	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8872	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8873	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8874	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8875	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8876	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8877	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8878	}
8879
8880	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8881	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8882	{
8883	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8884	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8885	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8886	\|\| pCtx->ds.Attr.n.u4Type > 11
8887	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8888	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8889	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8890	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8891	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8892	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8893	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8894	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8895	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8896	}
8897	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8898	{
8899	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8900	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8901	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8902	\|\| pCtx->es.Attr.n.u4Type > 11
8903	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8904	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8905	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8906	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8907	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8908	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8909	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8910	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8911	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8912	}
8913	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8914	{
8915	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8916	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8917	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8918	\|\| pCtx->fs.Attr.n.u4Type > 11
8919	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8920	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8921	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8922	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8923	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8924	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8925	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8926	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8927	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8928	}
8929	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8930	{
8931	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8932	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8933	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8934	\|\| pCtx->gs.Attr.n.u4Type > 11
8935	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8936	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8937	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8938	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8939	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8940	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8941	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8942	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8943	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8944	}
8945	/* 64-bit capable CPUs. */
8946	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8947	if (HMVMX_IS_64BIT_HOST_MODE())
8948	{
8949	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8950	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8951	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8952	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8953	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8954	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8955	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8956	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8957	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8958	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8959	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8960	}
8961	#endif
8962	}
8963	else
8964	{
8965	/* V86 mode checks. */
8966	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8967	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8968	{
8969	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8970	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8971	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8972	}
8973	else
8974	{
8975	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8976	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8977	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8978	}
8979
8980	/* CS */
8981	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8982	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8983	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8984	/* SS */
8985	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8986	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8987	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8988	/* DS */
8989	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8990	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8991	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8992	/* ES */
8993	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8994	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8995	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8996	/* FS */
8997	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8998	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8999	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9000	/* GS */
9001	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9002	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9003	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9004	/* 64-bit capable CPUs. */
9005	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9006	if (HMVMX_IS_64BIT_HOST_MODE())
9007	{
9008	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9009	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9010	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9011	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9012	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9013	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9014	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9015	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9016	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9017	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9018	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9019	}
9020	#endif
9021	}
9022
9023	/*
9024	* TR.
9025	*/
9026	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9027	/* 64-bit capable CPUs. */
9028	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9029	if (HMVMX_IS_64BIT_HOST_MODE())
9030	{
9031	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9032	}
9033	#endif
9034	if (fLongModeGuest)
9035	{
9036	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9037	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9038	}
9039	else
9040	{
9041	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9042	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9043	VMX_IGS_TR_ATTR_TYPE_INVALID);
9044	}
9045	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9046	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9047	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9048	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9049	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9050	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9051	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9052	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9053
9054	/*
9055	* GDTR and IDTR.
9056	*/
9057	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9058	if (HMVMX_IS_64BIT_HOST_MODE())
9059	{
9060	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9061	AssertRCBreak(rc);
9062	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9063
9064	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9065	AssertRCBreak(rc);
9066	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9067	}
9068	#endif
9069
9070	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9071	AssertRCBreak(rc);
9072	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9073
9074	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9075	AssertRCBreak(rc);
9076	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9077
9078	/*
9079	* Guest Non-Register State.
9080	*/
9081	/* Activity State. */
9082	uint32_t u32ActivityState;
9083	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9084	AssertRCBreak(rc);
9085	HMVMX_CHECK_BREAK( !u32ActivityState
9086	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9087	VMX_IGS_ACTIVITY_STATE_INVALID);
9088	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9089	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9090	uint32_t u32IntrState;
9091	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9092	AssertRCBreak(rc);
9093	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9094	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9095	{
9096	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9097	}
9098
9099	/** @todo Activity state and injecting interrupts. Left as a todo since we
9100	* currently don't use activity states but ACTIVE. */
9101
9102	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9103	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9104
9105	/* Guest interruptibility-state. */
9106	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9107	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9108	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9109	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9110	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9111	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9112	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9113	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9114	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9115	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9116	{
9117	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9118	{
9119	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9120	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9121	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9122	}
9123	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9124	{
9125	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9126	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9127	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9128	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9129	}
9130	}
9131	/** @todo Assumes the processor is not in SMM. */
9132	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9133	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9134	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9135	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9136	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9137	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9138	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9139	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9140	{
9141	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9142	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9143	}
9144
9145	/* Pending debug exceptions. */
9146	if (HMVMX_IS_64BIT_HOST_MODE())
9147	{
9148	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9149	AssertRCBreak(rc);
9150	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9151	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9152	u32Val = u64Val; /* For pending debug exceptions checks below. */
9153	}
9154	else
9155	{
9156	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9157	AssertRCBreak(rc);
9158	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9159	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9160	}
9161
9162	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9163	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9164	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9165	{
9166	if ( (u32Eflags & X86_EFL_TF)
9167	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9168	{
9169	/* Bit 14 is PendingDebug.BS. */
9170	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9171	}
9172	if ( !(u32Eflags & X86_EFL_TF)
9173	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9174	{
9175	/* Bit 14 is PendingDebug.BS. */
9176	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9177	}
9178	}
9179
9180	/* VMCS link pointer. */
9181	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9182	AssertRCBreak(rc);
9183	if (u64Val != UINT64_C(0xffffffffffffffff))
9184	{
9185	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9186	/** @todo Bits beyond the processor's physical-address width MBZ. */
9187	/** @todo 32-bit located in memory referenced by value of this field (as a
9188	* physical address) must contain the processor's VMCS revision ID. */
9189	/** @todo SMM checks. */
9190	}
9191
9192	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9193	* not using Nested Paging? */
9194	if ( pVM->hm.s.fNestedPaging
9195	&& !fLongModeGuest
9196	&& CPUMIsGuestInPAEModeEx(pCtx))
9197	{
9198	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9199	AssertRCBreak(rc);
9200	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9201
9202	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9203	AssertRCBreak(rc);
9204	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9205
9206	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9207	AssertRCBreak(rc);
9208	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9209
9210	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9211	AssertRCBreak(rc);
9212	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9213	}
9214
9215	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9216	if (uError == VMX_IGS_ERROR)
9217	uError = VMX_IGS_REASON_NOT_FOUND;
9218	} while (0);
9219
9220	pVCpu->hm.s.u32HMError = uError;
9221	return uError;
9222
9223	#undef HMVMX_ERROR_BREAK
9224	#undef HMVMX_CHECK_BREAK
9225	}
9226
9227	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9228	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9229	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9230
9231	/** @name VM-exit handlers.
9232	* @{
9233	*/
9234
9235	/**
9236	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9237	*/
9238	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9239	{
9240	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9241	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9242	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9243	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9244	return VINF_SUCCESS;
9245	return VINF_EM_RAW_INTERRUPT;
9246	}
9247
9248
9249	/**
9250	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9251	*/
9252	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9253	{
9254	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9255	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9256
9257	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9258	AssertRCReturn(rc, rc);
9259
9260	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9261	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9262	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9263	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9264
9265	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9266	{
9267	/*
9268	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9269	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9270	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9271	*
9272	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9273	*/
9274	VMXDispatchHostNmi();
9275	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9276	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9277	return VINF_SUCCESS;
9278	}
9279
9280	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9281	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9282	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9283	{
9284	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9285	return VINF_SUCCESS;
9286	}
9287	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9288	{
9289	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9290	return rc;
9291	}
9292
9293	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9294	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9295	switch (uIntType)
9296	{
9297	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9298	Assert(uVector == X86_XCPT_DB);
9299	/* no break */
9300	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9301	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9302	/* no break */
9303	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9304	{
9305	switch (uVector)
9306	{
9307	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9308	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9309	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9310	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9311	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9312	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9313	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9314	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9315	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9316	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9317	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9318	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9319	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9320	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9321	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9322	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9323	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9324	#endif
9325	default:
9326	{
9327	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9328	AssertRCReturn(rc, rc);
9329
9330	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9331	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9332	{
9333	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9334	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9335	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9336
9337	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9338	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9339	AssertRCReturn(rc, rc);
9340	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9341	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9342	0 /* GCPtrFaultAddress */);
9343	AssertRCReturn(rc, rc);
9344	}
9345	else
9346	{
9347	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9348	pVCpu->hm.s.u32HMError = uVector;
9349	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9350	}
9351	break;
9352	}
9353	}
9354	break;
9355	}
9356
9357	default:
9358	{
9359	pVCpu->hm.s.u32HMError = uExitIntInfo;
9360	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9361	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9362	break;
9363	}
9364	}
9365	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9366	return rc;
9367	}
9368
9369
9370	/**
9371	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9372	*/
9373	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9374	{
9375	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9376
9377	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9378	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9379	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9380	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9381	AssertRCReturn(rc, rc);
9382
9383	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9385	return VINF_SUCCESS;
9386	}
9387
9388
9389	/**
9390	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9391	*/
9392	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9393	{
9394	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9395	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9396	HMVMX_RETURN_UNEXPECTED_EXIT();
9397	}
9398
9399
9400	/**
9401	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9402	*/
9403	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9404	{
9405	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9406	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9407	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9408	}
9409
9410
9411	/**
9412	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9413	*/
9414	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9415	{
9416	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9418	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9419	}
9420
9421
9422	/**
9423	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9424	*/
9425	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9426	{
9427	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9428	PVM pVM = pVCpu->CTX_SUFF(pVM);
9429	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9430	if (RT_LIKELY(rc == VINF_SUCCESS))
9431	{
9432	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9433	Assert(pVmxTransient->cbInstr == 2);
9434	}
9435	else
9436	{
9437	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9438	rc = VERR_EM_INTERPRETER;
9439	}
9440	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9441	return rc;
9442	}
9443
9444
9445	/**
9446	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9447	*/
9448	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9449	{
9450	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9451	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9452	AssertRCReturn(rc, rc);
9453
9454	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9455	return VINF_EM_RAW_EMULATE_INSTR;
9456
9457	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9458	HMVMX_RETURN_UNEXPECTED_EXIT();
9459	}
9460
9461
9462	/**
9463	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9464	*/
9465	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9466	{
9467	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9468	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9469	AssertRCReturn(rc, rc);
9470
9471	PVM pVM = pVCpu->CTX_SUFF(pVM);
9472	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9473	if (RT_LIKELY(rc == VINF_SUCCESS))
9474	{
9475	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9476	Assert(pVmxTransient->cbInstr == 2);
9477	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9478	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9479	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9480	}
9481	else
9482	{
9483	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9484	rc = VERR_EM_INTERPRETER;
9485	}
9486	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9487	return rc;
9488	}
9489
9490
9491	/**
9492	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9493	*/
9494	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9495	{
9496	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9497	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9498	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9499	AssertRCReturn(rc, rc);
9500
9501	PVM pVM = pVCpu->CTX_SUFF(pVM);
9502	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9503	if (RT_LIKELY(rc == VINF_SUCCESS))
9504	{
9505	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9506	Assert(pVmxTransient->cbInstr == 3);
9507	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9508	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9509	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9510	}
9511	else
9512	{
9513	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9514	rc = VERR_EM_INTERPRETER;
9515	}
9516	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9517	return rc;
9518	}
9519
9520
9521	/**
9522	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9523	*/
9524	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9525	{
9526	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9527	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9528	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9529	AssertRCReturn(rc, rc);
9530
9531	PVM pVM = pVCpu->CTX_SUFF(pVM);
9532	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9533	if (RT_LIKELY(rc == VINF_SUCCESS))
9534	{
9535	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9536	Assert(pVmxTransient->cbInstr == 2);
9537	}
9538	else
9539	{
9540	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9541	rc = VERR_EM_INTERPRETER;
9542	}
9543	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9544	return rc;
9545	}
9546
9547
9548	/**
9549	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9550	*/
9551	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9552	{
9553	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9554	PVM pVM = pVCpu->CTX_SUFF(pVM);
9555	Assert(!pVM->hm.s.fNestedPaging);
9556
9557	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9558	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9559	AssertRCReturn(rc, rc);
9560
9561	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9562	rc = VBOXSTRICTRC_VAL(rc2);
9563	if (RT_LIKELY(rc == VINF_SUCCESS))
9564	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9565	else
9566	{
9567	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9568	pVmxTransient->uExitQualification, rc));
9569	}
9570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9571	return rc;
9572	}
9573
9574
9575	/**
9576	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9577	*/
9578	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9579	{
9580	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9581	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9582	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9583	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9584	AssertRCReturn(rc, rc);
9585
9586	PVM pVM = pVCpu->CTX_SUFF(pVM);
9587	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9588	if (RT_LIKELY(rc == VINF_SUCCESS))
9589	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9590	else
9591	{
9592	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9593	rc = VERR_EM_INTERPRETER;
9594	}
9595	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9596	return rc;
9597	}
9598
9599
9600	/**
9601	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9602	*/
9603	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9604	{
9605	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9606	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9607	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9608	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9609	AssertRCReturn(rc, rc);
9610
9611	PVM pVM = pVCpu->CTX_SUFF(pVM);
9612	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9613	rc = VBOXSTRICTRC_VAL(rc2);
9614	if (RT_LIKELY( rc == VINF_SUCCESS
9615	\|\| rc == VINF_EM_HALT))
9616	{
9617	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9618	AssertRCReturn(rc3, rc3);
9619
9620	if ( rc == VINF_EM_HALT
9621	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9622	{
9623	rc = VINF_SUCCESS;
9624	}
9625	}
9626	else
9627	{
9628	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9629	rc = VERR_EM_INTERPRETER;
9630	}
9631	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9632	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9633	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9634	return rc;
9635	}
9636
9637
9638	/**
9639	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9640	*/
9641	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9642	{
9643	/*
9644	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9645	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9646	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9647	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9648	*/
9649	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9650	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9651	HMVMX_RETURN_UNEXPECTED_EXIT();
9652	}
9653
9654
9655	/**
9656	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9657	*/
9658	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9659	{
9660	/*
9661	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9662	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9663	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9664	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9665	*/
9666	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9667	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9668	HMVMX_RETURN_UNEXPECTED_EXIT();
9669	}
9670
9671
9672	/**
9673	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9674	*/
9675	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9676	{
9677	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9678	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9679	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9680	HMVMX_RETURN_UNEXPECTED_EXIT();
9681	}
9682
9683
9684	/**
9685	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9686	*/
9687	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9688	{
9689	/*
9690	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9691	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9692	* See Intel spec. 25.3 "Other Causes of VM-exits".
9693	*/
9694	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9695	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9696	HMVMX_RETURN_UNEXPECTED_EXIT();
9697	}
9698
9699
9700	/**
9701	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9702	* VM-exit.
9703	*/
9704	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9705	{
9706	/*
9707	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9708	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9709	*
9710	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9711	* See Intel spec. "23.8 Restrictions on VMX operation".
9712	*/
9713	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9714	return VINF_SUCCESS;
9715	}
9716
9717
9718	/**
9719	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9720	* VM-exit.
9721	*/
9722	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9723	{
9724	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9725	return VINF_EM_RESET;
9726	}
9727
9728
9729	/**
9730	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9731	*/
9732	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9733	{
9734	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9735	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9736	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9737	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9738	AssertRCReturn(rc, rc);
9739
9740	pMixedCtx->rip++;
9741	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9742	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9743	rc = VINF_SUCCESS;
9744	else
9745	rc = VINF_EM_HALT;
9746
9747	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9748	return rc;
9749	}
9750
9751
9752	/**
9753	* VM-exit handler for instructions that result in a #UD exception delivered to
9754	* the guest.
9755	*/
9756	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9757	{
9758	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9759	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9760	return VINF_SUCCESS;
9761	}
9762
9763
9764	/**
9765	* VM-exit handler for expiry of the VMX preemption timer.
9766	*/
9767	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9768	{
9769	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9770
9771	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9772	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9773
9774	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9775	PVM pVM = pVCpu->CTX_SUFF(pVM);
9776	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9777	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9778	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9779	}
9780
9781
9782	/**
9783	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9784	*/
9785	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9786	{
9787	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9788
9789	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9790	/** @todo check if XSETBV is supported by the recompiler. */
9791	return VERR_EM_INTERPRETER;
9792	}
9793
9794
9795	/**
9796	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9797	*/
9798	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9799	{
9800	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9801
9802	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9803	/** @todo implement EMInterpretInvpcid() */
9804	return VERR_EM_INTERPRETER;
9805	}
9806
9807
9808	/**
9809	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9810	* Error VM-exit.
9811	*/
9812	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9813	{
9814	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9815	AssertRCReturn(rc, rc);
9816
9817	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9818	NOREF(uInvalidReason);
9819
9820	#ifdef VBOX_STRICT
9821	uint32_t uIntrState;
9822	HMVMXHCUINTREG uHCReg;
9823	uint64_t u64Val;
9824	uint32_t u32Val;
9825
9826	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9827	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9828	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9829	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9830	AssertRCReturn(rc, rc);
9831
9832	Log4(("uInvalidReason %u\n", uInvalidReason));
9833	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9834	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9835	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9836	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9837
9838	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9839	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9840	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9841	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9842	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9843	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9844	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9845	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9846	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9847	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9848	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9849	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9850	#else
9851	NOREF(pVmxTransient);
9852	#endif
9853
9854	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9855	return VERR_VMX_INVALID_GUEST_STATE;
9856	}
9857
9858
9859	/**
9860	* VM-exit handler for VM-entry failure due to an MSR-load
9861	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9862	*/
9863	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9864	{
9865	NOREF(pVmxTransient);
9866	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9867	HMVMX_RETURN_UNEXPECTED_EXIT();
9868	}
9869
9870
9871	/**
9872	* VM-exit handler for VM-entry failure due to a machine-check event
9873	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9874	*/
9875	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9876	{
9877	NOREF(pVmxTransient);
9878	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9879	HMVMX_RETURN_UNEXPECTED_EXIT();
9880	}
9881
9882
9883	/**
9884	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9885	* theory.
9886	*/
9887	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9888	{
9889	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9890	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9891	return VERR_VMX_UNDEFINED_EXIT_CODE;
9892	}
9893
9894
9895	/**
9896	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9897	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9898	* Conditional VM-exit.
9899	*/
9900	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9901	{
9902	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9903
9904	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9905	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9906	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9907	return VERR_EM_INTERPRETER;
9908	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9909	HMVMX_RETURN_UNEXPECTED_EXIT();
9910	}
9911
9912
9913	/**
9914	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9915	*/
9916	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9917	{
9918	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9919
9920	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9921	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9922	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9923	return VERR_EM_INTERPRETER;
9924	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9925	HMVMX_RETURN_UNEXPECTED_EXIT();
9926	}
9927
9928
9929	/**
9930	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9931	*/
9932	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9933	{
9934	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9935
9936	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9937	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9938	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9939	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9940	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9941	{
9942	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9943	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9944	}
9945	AssertRCReturn(rc, rc);
9946	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9947
9948	#ifdef VBOX_STRICT
9949	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9950	{
9951	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9952	{
9953	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9954	HMVMX_RETURN_UNEXPECTED_EXIT();
9955	}
9956	# if HC_ARCH_BITS == 64
9957	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
9958	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
9959	{
9960	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
9961	HMVMX_RETURN_UNEXPECTED_EXIT();
9962	}
9963	# endif
9964	}
9965	#endif
9966
9967	PVM pVM = pVCpu->CTX_SUFF(pVM);
9968	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9969	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9970	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9971	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9972
9973	if (RT_LIKELY(rc == VINF_SUCCESS))
9974	{
9975	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9976	Assert(pVmxTransient->cbInstr == 2);
9977	}
9978	return rc;
9979	}
9980
9981
9982	/**
9983	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9984	*/
9985	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9986	{
9987	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9988	PVM pVM = pVCpu->CTX_SUFF(pVM);
9989	int rc = VINF_SUCCESS;
9990
9991	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9992	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9993	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9994	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9995	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9996	{
9997	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9998	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9999	}
10000	AssertRCReturn(rc, rc);
10001	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10002
10003	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10004	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10005	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10006
10007	if (RT_LIKELY(rc == VINF_SUCCESS))
10008	{
10009	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10010
10011	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10012	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10013	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10014	{
10015	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10016	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10017	EMInterpretWrmsr() changes it. */
10018	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10019	}
10020	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10021	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10022
10023	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10024	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10025	{
10026	switch (pMixedCtx->ecx)
10027	{
10028	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10029	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10030	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10031	case MSR_K8_FS_BASE: /* no break */
10032	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10033	default:
10034	{
10035	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10036	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10037	#if HC_ARCH_BITS == 64
10038	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10039	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10040	#endif
10041	break;
10042	}
10043	}
10044	}
10045	#ifdef VBOX_STRICT
10046	else
10047	{
10048	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10049	switch (pMixedCtx->ecx)
10050	{
10051	case MSR_IA32_SYSENTER_CS:
10052	case MSR_IA32_SYSENTER_EIP:
10053	case MSR_IA32_SYSENTER_ESP:
10054	case MSR_K8_FS_BASE:
10055	case MSR_K8_GS_BASE:
10056	{
10057	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10058	HMVMX_RETURN_UNEXPECTED_EXIT();
10059	}
10060
10061	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10062	default:
10063	{
10064	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10065	{
10066	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10067	pMixedCtx->ecx));
10068	HMVMX_RETURN_UNEXPECTED_EXIT();
10069	}
10070
10071	#if HC_ARCH_BITS == 64
10072	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10073	{
10074	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10075	HMVMX_RETURN_UNEXPECTED_EXIT();
10076	}
10077	#endif
10078	break;
10079	}
10080	}
10081	}
10082	#endif /* VBOX_STRICT */
10083	}
10084	return rc;
10085	}
10086
10087
10088	/**
10089	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10090	*/
10091	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10092	{
10093	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10094
10095	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10097	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10098	return VERR_EM_INTERPRETER;
10099	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10100	HMVMX_RETURN_UNEXPECTED_EXIT();
10101	}
10102
10103
10104	/**
10105	* VM-exit handler for when the TPR value is lowered below the specified
10106	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10107	*/
10108	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10109	{
10110	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10111	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10112
10113	/*
10114	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10115	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10116	* resume guest execution.
10117	*/
10118	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10119	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10120	return VINF_SUCCESS;
10121	}
10122
10123
10124	/**
10125	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10126	* VM-exit.
10127	*
10128	* @retval VINF_SUCCESS when guest execution can continue.
10129	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10130	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10131	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10132	* recompiler.
10133	*/
10134	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10135	{
10136	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10137	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10138	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10139	AssertRCReturn(rc, rc);
10140
10141	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10142	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10143	PVM pVM = pVCpu->CTX_SUFF(pVM);
10144	switch (uAccessType)
10145	{
10146	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10147	{
10148	#if 0
10149	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10150	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10151	#else
10152	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10153	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10154	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10155	#endif
10156	AssertRCReturn(rc, rc);
10157
10158	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10159	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10160	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10161	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10162
10163	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10164	{
10165	case 0: /* CR0 */
10166	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10167	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10168	break;
10169	case 2: /* CR2 */
10170	/* Nothing to do here, CR2 it's not part of the VMCS. */
10171	break;
10172	case 3: /* CR3 */
10173	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10174	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10175	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10176	break;
10177	case 4: /* CR4 */
10178	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10179	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10180	break;
10181	case 8: /* CR8 */
10182	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10183	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10184	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10185	break;
10186	default:
10187	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10188	break;
10189	}
10190
10191	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10192	break;
10193	}
10194
10195	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10196	{
10197	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10198	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10199	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10200	AssertRCReturn(rc, rc);
10201	Assert( !pVM->hm.s.fNestedPaging
10202	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10203	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10204
10205	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10206	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10207	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10208
10209	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10210	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10211	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10212	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10213	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10214	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10215	break;
10216	}
10217
10218	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10219	{
10220	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10221	AssertRCReturn(rc, rc);
10222	rc = EMInterpretCLTS(pVM, pVCpu);
10223	AssertRCReturn(rc, rc);
10224	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10226	Log4(("CRX CLTS write rc=%d\n", rc));
10227	break;
10228	}
10229
10230	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10231	{
10232	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10233	AssertRCReturn(rc, rc);
10234	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10235	if (RT_LIKELY(rc == VINF_SUCCESS))
10236	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10238	Log4(("CRX LMSW write rc=%d\n", rc));
10239	break;
10240	}
10241
10242	default:
10243	{
10244	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10245	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10246	}
10247	}
10248
10249	/* Validate possible error codes. */
10250	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10251	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10252	if (RT_SUCCESS(rc))
10253	{
10254	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10255	AssertRCReturn(rc2, rc2);
10256	}
10257
10258	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10259	return rc;
10260	}
10261
10262
10263	/**
10264	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10265	* VM-exit.
10266	*/
10267	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10268	{
10269	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10270	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10271
10272	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10273	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10274	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10275	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10276	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10277	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10278	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10279	AssertRCReturn(rc2, rc2);
10280
10281	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10282	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10283	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10284	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10285	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10286	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10287	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10288
10289	/* I/O operation lookup arrays. */
10290	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10291	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10292
10293	VBOXSTRICTRC rcStrict;
10294	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10295	const uint32_t cbInstr = pVmxTransient->cbInstr;
10296	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10297	PVM pVM = pVCpu->CTX_SUFF(pVM);
10298	if (fIOString)
10299	{
10300	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10301	/*
10302	* INS/OUTS - I/O String instruction.
10303	*
10304	* Use instruction-information if available, otherwise fall back on
10305	* interpreting the instruction.
10306	*/
10307	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10308	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10309	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10310	{
10311	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10312	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10313	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10314	AssertRCReturn(rc2, rc2);
10315	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10316	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10317	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10318	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10319	if (fIOWrite)
10320	{
10321	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10322	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10323	}
10324	else
10325	{
10326	/*
10327	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10328	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10329	* See Intel Instruction spec. for "INS".
10330	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10331	*/
10332	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10333	}
10334	}
10335	else
10336	{
10337	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10338	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10339	AssertRCReturn(rc2, rc2);
10340	rcStrict = IEMExecOne(pVCpu);
10341	}
10342	/** @todo IEM needs to be setting these flags somehow. */
10343	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10344	fUpdateRipAlready = true;
10345	#else
10346	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10347	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10348	if (RT_SUCCESS(rcStrict))
10349	{
10350	if (fIOWrite)
10351	{
10352	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10353	(DISCPUMODE)pDis->uAddrMode, cbValue);
10354	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10355	}
10356	else
10357	{
10358	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10359	(DISCPUMODE)pDis->uAddrMode, cbValue);
10360	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10361	}
10362	}
10363	else
10364	{
10365	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10366	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10367	}
10368	#endif
10369	}
10370	else
10371	{
10372	/*
10373	* IN/OUT - I/O instruction.
10374	*/
10375	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10376	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10377	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10378	if (fIOWrite)
10379	{
10380	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10381	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10382	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10383	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10384	}
10385	else
10386	{
10387	uint32_t u32Result = 0;
10388	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10389	if (IOM_SUCCESS(rcStrict))
10390	{
10391	/* Save result of I/O IN instr. in AL/AX/EAX. */
10392	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10393	}
10394	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10395	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10397	}
10398	}
10399
10400	if (IOM_SUCCESS(rcStrict))
10401	{
10402	if (!fUpdateRipAlready)
10403	{
10404	pMixedCtx->rip += cbInstr;
10405	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10406	}
10407
10408	/*
10409	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10410	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10411	*/
10412	if (fIOString)
10413	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10414
10415	/*
10416	* If any I/O breakpoints are armed, we need to check if one triggered
10417	* and take appropriate action.
10418	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10419	*/
10420	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10421	AssertRCReturn(rc2, rc2);
10422
10423	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10424	* execution engines about whether hyper BPs and such are pending. */
10425	uint32_t const uDr7 = pMixedCtx->dr[7];
10426	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10427	&& X86_DR7_ANY_RW_IO(uDr7)
10428	&& (pMixedCtx->cr4 & X86_CR4_DE))
10429	\|\| DBGFBpIsHwIoArmed(pVM)))
10430	{
10431	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10432
10433	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10434	VMMRZCallRing3Disable(pVCpu);
10435	HM_DISABLE_PREEMPT_IF_NEEDED();
10436
10437	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10438
10439	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10440	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10441	{
10442	/* Raise #DB. */
10443	if (fIsGuestDbgActive)
10444	ASMSetDR6(pMixedCtx->dr[6]);
10445	if (pMixedCtx->dr[7] != uDr7)
10446	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10447
10448	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10449	}
10450	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10451	else if ( rcStrict2 != VINF_SUCCESS
10452	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10453	rcStrict = rcStrict2;
10454
10455	HM_RESTORE_PREEMPT_IF_NEEDED();
10456	VMMRZCallRing3Enable(pVCpu);
10457	}
10458	}
10459
10460	#ifdef DEBUG
10461	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10462	Assert(!fIOWrite);
10463	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10464	Assert(fIOWrite);
10465	else
10466	{
10467	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10468	* statuses, that the VMM device and some others may return. See
10469	* IOM_SUCCESS() for guidance. */
10470	AssertMsg( RT_FAILURE(rcStrict)
10471	\|\| rcStrict == VINF_SUCCESS
10472	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10473	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10474	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10475	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10476	}
10477	#endif
10478
10479	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10480	return VBOXSTRICTRC_TODO(rcStrict);
10481	}
10482
10483
10484	/**
10485	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10486	* VM-exit.
10487	*/
10488	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10489	{
10490	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10491
10492	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10493	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10494	AssertRCReturn(rc, rc);
10495	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10496	{
10497	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10498	AssertRCReturn(rc, rc);
10499	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10500	{
10501	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10502
10503	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10504	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10505	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10506	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10507	{
10508	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10509	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10510
10511	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10512	Assert(!pVCpu->hm.s.Event.fPending);
10513	pVCpu->hm.s.Event.fPending = true;
10514	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10515	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10516	AssertRCReturn(rc, rc);
10517	if (fErrorCodeValid)
10518	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10519	else
10520	pVCpu->hm.s.Event.u32ErrCode = 0;
10521	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10522	&& uVector == X86_XCPT_PF)
10523	{
10524	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10525	}
10526
10527	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10528	}
10529	}
10530	}
10531
10532	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10533	* emulation. */
10534	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10535	return VERR_EM_INTERPRETER;
10536	}
10537
10538
10539	/**
10540	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10541	*/
10542	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10543	{
10544	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10545	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10546	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10547	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10548	AssertRCReturn(rc, rc);
10549	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10550	return VINF_EM_DBG_STEPPED;
10551	}
10552
10553
10554	/**
10555	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10556	*/
10557	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10558	{
10559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10560
10561	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10562	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10563	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10564	return VINF_SUCCESS;
10565	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10566	return rc;
10567
10568	#if 0
10569	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10570	* just sync the whole thing. */
10571	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10572	#else
10573	/* Aggressive state sync. for now. */
10574	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10575	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10576	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10577	#endif
10578	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10579	AssertRCReturn(rc, rc);
10580
10581	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10582	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10583	switch (uAccessType)
10584	{
10585	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10586	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10587	{
10588	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10589	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10590	{
10591	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10592	}
10593
10594	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10595	GCPhys &= PAGE_BASE_GC_MASK;
10596	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10597	PVM pVM = pVCpu->CTX_SUFF(pVM);
10598	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10599	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10600
10601	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10602	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10603	CPUMCTX2CORE(pMixedCtx), GCPhys);
10604	rc = VBOXSTRICTRC_VAL(rc2);
10605	Log4(("ApicAccess rc=%d\n", rc));
10606	if ( rc == VINF_SUCCESS
10607	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10608	\|\| rc == VERR_PAGE_NOT_PRESENT)
10609	{
10610	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10611	\| HM_CHANGED_GUEST_RSP
10612	\| HM_CHANGED_GUEST_RFLAGS
10613	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10614	rc = VINF_SUCCESS;
10615	}
10616	break;
10617	}
10618
10619	default:
10620	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10621	rc = VINF_EM_RAW_EMULATE_INSTR;
10622	break;
10623	}
10624
10625	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10626	return rc;
10627	}
10628
10629
10630	/**
10631	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10632	* VM-exit.
10633	*/
10634	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10635	{
10636	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10637
10638	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10639	if (pVmxTransient->fWasGuestDebugStateActive)
10640	{
10641	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10642	HMVMX_RETURN_UNEXPECTED_EXIT();
10643	}
10644
10645	int rc = VERR_INTERNAL_ERROR_5;
10646	if ( !DBGFIsStepping(pVCpu)
10647	&& !pVCpu->hm.s.fSingleInstruction
10648	&& !pVmxTransient->fWasHyperDebugStateActive)
10649	{
10650	/* Don't intercept MOV DRx and #DB any more. */
10651	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10652	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10653	AssertRCReturn(rc, rc);
10654
10655	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10656	{
10657	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10658	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10659	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10660	AssertRCReturn(rc, rc);
10661	#endif
10662	}
10663
10664	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10665	VMMRZCallRing3Disable(pVCpu);
10666	HM_DISABLE_PREEMPT_IF_NEEDED();
10667
10668	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10669	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10670	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10671
10672	HM_RESTORE_PREEMPT_IF_NEEDED();
10673	VMMRZCallRing3Enable(pVCpu);
10674
10675	#ifdef VBOX_WITH_STATISTICS
10676	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10677	AssertRCReturn(rc, rc);
10678	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10679	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10680	else
10681	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10682	#endif
10683	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10684	return VINF_SUCCESS;
10685	}
10686
10687	/*
10688	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10689	* Update the segment registers and DR7 from the CPU.
10690	*/
10691	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10692	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10693	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10694	AssertRCReturn(rc, rc);
10695	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10696
10697	PVM pVM = pVCpu->CTX_SUFF(pVM);
10698	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10699	{
10700	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10701	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10702	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10703	if (RT_SUCCESS(rc))
10704	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10705	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10706	}
10707	else
10708	{
10709	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10710	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10711	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10712	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10713	}
10714
10715	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10716	if (RT_SUCCESS(rc))
10717	{
10718	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10719	AssertRCReturn(rc2, rc2);
10720	}
10721	return rc;
10722	}
10723
10724
10725	/**
10726	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10727	* Conditional VM-exit.
10728	*/
10729	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10730	{
10731	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10732	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10733
10734	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10735	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10736	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10737	return VINF_SUCCESS;
10738	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10739	return rc;
10740
10741	RTGCPHYS GCPhys = 0;
10742	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10743
10744	#if 0
10745	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10746	#else
10747	/* Aggressive state sync. for now. */
10748	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10749	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10750	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10751	#endif
10752	AssertRCReturn(rc, rc);
10753
10754	/*
10755	* If we succeed, resume guest execution.
10756	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10757	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10758	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10759	* weird case. See @bugref{6043}.
10760	*/
10761	PVM pVM = pVCpu->CTX_SUFF(pVM);
10762	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10763	rc = VBOXSTRICTRC_VAL(rc2);
10764	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10765	if ( rc == VINF_SUCCESS
10766	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10767	\|\| rc == VERR_PAGE_NOT_PRESENT)
10768	{
10769	/* Successfully handled MMIO operation. */
10770	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10771	\| HM_CHANGED_GUEST_RSP
10772	\| HM_CHANGED_GUEST_RFLAGS
10773	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10774	rc = VINF_SUCCESS;
10775	}
10776	return rc;
10777	}
10778
10779
10780	/**
10781	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10782	* VM-exit.
10783	*/
10784	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10785	{
10786	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10787	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10788
10789	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10790	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10791	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10792	return VINF_SUCCESS;
10793	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10794	return rc;
10795
10796	RTGCPHYS GCPhys = 0;
10797	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10798	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10799	#if 0
10800	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10801	#else
10802	/* Aggressive state sync. for now. */
10803	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10804	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10805	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10806	#endif
10807	AssertRCReturn(rc, rc);
10808
10809	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10810	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10811
10812	RTGCUINT uErrorCode = 0;
10813	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10814	uErrorCode \|= X86_TRAP_PF_ID;
10815	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10816	uErrorCode \|= X86_TRAP_PF_RW;
10817	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10818	uErrorCode \|= X86_TRAP_PF_P;
10819
10820	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10821
10822	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10823	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10824
10825	/* Handle the pagefault trap for the nested shadow table. */
10826	PVM pVM = pVCpu->CTX_SUFF(pVM);
10827	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10828	TRPMResetTrap(pVCpu);
10829
10830	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10831	if ( rc == VINF_SUCCESS
10832	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10833	\|\| rc == VERR_PAGE_NOT_PRESENT)
10834	{
10835	/* Successfully synced our nested page tables. */
10836	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10837	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10838	\| HM_CHANGED_GUEST_RSP
10839	\| HM_CHANGED_GUEST_RFLAGS);
10840	return VINF_SUCCESS;
10841	}
10842
10843	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10844	return rc;
10845	}
10846
10847	/** @} */
10848
10849	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10850	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10851	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10852
10853	/** @name VM-exit exception handlers.
10854	* @{
10855	*/
10856
10857	/**
10858	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10859	*/
10860	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10861	{
10862	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10863	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10864
10865	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10866	AssertRCReturn(rc, rc);
10867
10868	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10869	{
10870	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10871	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10872
10873	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10874	* provides VM-exit instruction length. If this causes problem later,
10875	* disassemble the instruction like it's done on AMD-V. */
10876	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10877	AssertRCReturn(rc2, rc2);
10878	return rc;
10879	}
10880
10881	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10882	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10883	return rc;
10884	}
10885
10886
10887	/**
10888	* VM-exit exception handler for #BP (Breakpoint exception).
10889	*/
10890	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10891	{
10892	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10893	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10894
10895	/** @todo Try optimize this by not saving the entire guest state unless
10896	* really needed. */
10897	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10898	AssertRCReturn(rc, rc);
10899
10900	PVM pVM = pVCpu->CTX_SUFF(pVM);
10901	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10902	if (rc == VINF_EM_RAW_GUEST_TRAP)
10903	{
10904	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10905	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10906	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10907	AssertRCReturn(rc, rc);
10908
10909	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10910	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10911	}
10912
10913	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10914	return rc;
10915	}
10916
10917
10918	/**
10919	* VM-exit exception handler for #DB (Debug exception).
10920	*/
10921	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10922	{
10923	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10924	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10925	Log6(("XcptDB\n"));
10926
10927	/*
10928	* Get the DR6-like values from the exit qualification and pass it to DBGF
10929	* for processing.
10930	*/
10931	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10932	AssertRCReturn(rc, rc);
10933
10934	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10935	uint64_t uDR6 = X86_DR6_INIT_VAL;
10936	uDR6 \|= ( pVmxTransient->uExitQualification
10937	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10938
10939	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10940	if (rc == VINF_EM_RAW_GUEST_TRAP)
10941	{
10942	/*
10943	* The exception was for the guest. Update DR6, DR7.GD and
10944	* IA32_DEBUGCTL.LBR before forwarding it.
10945	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10946	*/
10947	VMMRZCallRing3Disable(pVCpu);
10948	HM_DISABLE_PREEMPT_IF_NEEDED();
10949
10950	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10951	pMixedCtx->dr[6] \|= uDR6;
10952	if (CPUMIsGuestDebugStateActive(pVCpu))
10953	ASMSetDR6(pMixedCtx->dr[6]);
10954
10955	HM_RESTORE_PREEMPT_IF_NEEDED();
10956	VMMRZCallRing3Enable(pVCpu);
10957
10958	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10959	AssertRCReturn(rc, rc);
10960
10961	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10962	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10963
10964	/* Paranoia. */
10965	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10966	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10967
10968	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10969	AssertRCReturn(rc, rc);
10970
10971	/*
10972	* Raise #DB in the guest.
10973	*
10974	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
10975	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
10976	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
10977	*
10978	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
10979	*/
10980	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10981	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10982	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10983	AssertRCReturn(rc, rc);
10984	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10985	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10986	return VINF_SUCCESS;
10987	}
10988
10989	/*
10990	* Not a guest trap, must be a hypervisor related debug event then.
10991	* Update DR6 in case someone is interested in it.
10992	*/
10993	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10994	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10995	CPUMSetHyperDR6(pVCpu, uDR6);
10996
10997	return rc;
10998	}
10999
11000
11001	/**
11002	* VM-exit exception handler for #NM (Device-not-available exception: floating
11003	* point exception).
11004	*/
11005	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11006	{
11007	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11008
11009	/* We require CR0 and EFER. EFER is always up-to-date. */
11010	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11011	AssertRCReturn(rc, rc);
11012
11013	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11014	VMMRZCallRing3Disable(pVCpu);
11015	HM_DISABLE_PREEMPT_IF_NEEDED();
11016
11017	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11018	if (pVmxTransient->fWasGuestFPUStateActive)
11019	{
11020	rc = VINF_EM_RAW_GUEST_TRAP;
11021	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11022	}
11023	else
11024	{
11025	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11026	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11027	#endif
11028	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11029	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11030	}
11031
11032	HM_RESTORE_PREEMPT_IF_NEEDED();
11033	VMMRZCallRing3Enable(pVCpu);
11034
11035	if (rc == VINF_SUCCESS)
11036	{
11037	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11038	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11039	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11040	pVCpu->hm.s.fUseGuestFpu = true;
11041	}
11042	else
11043	{
11044	/* Forward #NM to the guest. */
11045	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11046	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11047	AssertRCReturn(rc, rc);
11048	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11049	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11050	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11051	}
11052
11053	return VINF_SUCCESS;
11054	}
11055
11056
11057	/**
11058	* VM-exit exception handler for #GP (General-protection exception).
11059	*
11060	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11061	*/
11062	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11063	{
11064	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11065	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11066
11067	int rc = VERR_INTERNAL_ERROR_5;
11068	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11069	{
11070	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11071	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11072	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11073	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11074	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11075	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11076	AssertRCReturn(rc, rc);
11077	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11078	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11079	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11080	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11081	return rc;
11082	#else
11083	/* We don't intercept #GP. */
11084	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11085	NOREF(pVmxTransient);
11086	return VERR_VMX_UNEXPECTED_EXCEPTION;
11087	#endif
11088	}
11089
11090	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11091	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11092
11093	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11094	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11095	AssertRCReturn(rc, rc);
11096
11097	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11098	uint32_t cbOp = 0;
11099	PVM pVM = pVCpu->CTX_SUFF(pVM);
11100	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11101	if (RT_SUCCESS(rc))
11102	{
11103	rc = VINF_SUCCESS;
11104	Assert(cbOp == pDis->cbInstr);
11105	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11106	switch (pDis->pCurInstr->uOpcode)
11107	{
11108	case OP_CLI:
11109	{
11110	pMixedCtx->eflags.Bits.u1IF = 0;
11111	pMixedCtx->rip += pDis->cbInstr;
11112	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11113	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11114	break;
11115	}
11116
11117	case OP_STI:
11118	{
11119	pMixedCtx->eflags.Bits.u1IF = 1;
11120	pMixedCtx->rip += pDis->cbInstr;
11121	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11122	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11123	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11125	break;
11126	}
11127
11128	case OP_HLT:
11129	{
11130	rc = VINF_EM_HALT;
11131	pMixedCtx->rip += pDis->cbInstr;
11132	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11133	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11134	break;
11135	}
11136
11137	case OP_POPF:
11138	{
11139	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11140	uint32_t cbParm = 0;
11141	uint32_t uMask = 0;
11142	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11143	{
11144	cbParm = 4;
11145	uMask = 0xffffffff;
11146	}
11147	else
11148	{
11149	cbParm = 2;
11150	uMask = 0xffff;
11151	}
11152
11153	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11154	RTGCPTR GCPtrStack = 0;
11155	X86EFLAGS Eflags;
11156	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11157	&GCPtrStack);
11158	if (RT_SUCCESS(rc))
11159	{
11160	Assert(sizeof(Eflags.u32) >= cbParm);
11161	Eflags.u32 = 0;
11162	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11163	}
11164	if (RT_FAILURE(rc))
11165	{
11166	rc = VERR_EM_INTERPRETER;
11167	break;
11168	}
11169	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11170	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11171	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11172	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11173	pMixedCtx->esp += cbParm;
11174	pMixedCtx->esp &= uMask;
11175	pMixedCtx->rip += pDis->cbInstr;
11176
11177	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11178	\| HM_CHANGED_GUEST_RSP
11179	\| HM_CHANGED_GUEST_RFLAGS);
11180	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11181	break;
11182	}
11183
11184	case OP_PUSHF:
11185	{
11186	uint32_t cbParm = 0;
11187	uint32_t uMask = 0;
11188	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11189	{
11190	cbParm = 4;
11191	uMask = 0xffffffff;
11192	}
11193	else
11194	{
11195	cbParm = 2;
11196	uMask = 0xffff;
11197	}
11198
11199	/* Get the stack pointer & push the contents of eflags onto the stack. */
11200	RTGCPTR GCPtrStack = 0;
11201	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11202	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11203	if (RT_FAILURE(rc))
11204	{
11205	rc = VERR_EM_INTERPRETER;
11206	break;
11207	}
11208	X86EFLAGS Eflags = pMixedCtx->eflags;
11209	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11210	Eflags.Bits.u1RF = 0;
11211	Eflags.Bits.u1VM = 0;
11212
11213	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11214	if (RT_FAILURE(rc))
11215	{
11216	rc = VERR_EM_INTERPRETER;
11217	break;
11218	}
11219	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11220	pMixedCtx->esp -= cbParm;
11221	pMixedCtx->esp &= uMask;
11222	pMixedCtx->rip += pDis->cbInstr;
11223	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11224	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11225	break;
11226	}
11227
11228	case OP_IRET:
11229	{
11230	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11231	* instruction reference. */
11232	RTGCPTR GCPtrStack = 0;
11233	uint32_t uMask = 0xffff;
11234	uint16_t aIretFrame[3];
11235	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11236	{
11237	rc = VERR_EM_INTERPRETER;
11238	break;
11239	}
11240	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11241	&GCPtrStack);
11242	if (RT_SUCCESS(rc))
11243	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11244	if (RT_FAILURE(rc))
11245	{
11246	rc = VERR_EM_INTERPRETER;
11247	break;
11248	}
11249	pMixedCtx->eip = 0;
11250	pMixedCtx->ip = aIretFrame[0];
11251	pMixedCtx->cs.Sel = aIretFrame[1];
11252	pMixedCtx->cs.ValidSel = aIretFrame[1];
11253	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11254	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11255	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11256	pMixedCtx->sp += sizeof(aIretFrame);
11257	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11258	\| HM_CHANGED_GUEST_SEGMENT_REGS
11259	\| HM_CHANGED_GUEST_RSP
11260	\| HM_CHANGED_GUEST_RFLAGS);
11261	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11262	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11263	break;
11264	}
11265
11266	case OP_INT:
11267	{
11268	uint16_t uVector = pDis->Param1.uValue & 0xff;
11269	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11270	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11271	break;
11272	}
11273
11274	case OP_INTO:
11275	{
11276	if (pMixedCtx->eflags.Bits.u1OF)
11277	{
11278	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11280	}
11281	break;
11282	}
11283
11284	default:
11285	{
11286	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11287	EMCODETYPE_SUPERVISOR);
11288	rc = VBOXSTRICTRC_VAL(rc2);
11289	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11290	Log4(("#GP rc=%Rrc\n", rc));
11291	break;
11292	}
11293	}
11294	}
11295	else
11296	rc = VERR_EM_INTERPRETER;
11297
11298	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11299	("#GP Unexpected rc=%Rrc\n", rc));
11300	return rc;
11301	}
11302
11303
11304	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11305	/**
11306	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11307	* the exception reported in the VMX transient structure back into the VM.
11308	*
11309	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11310	* up-to-date.
11311	*/
11312	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11313	{
11314	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11315
11316	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11317	hmR0VmxCheckExitDueToEventDelivery(). */
11318	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11319	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11320	AssertRCReturn(rc, rc);
11321	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11322
11323	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11324	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11325	return VINF_SUCCESS;
11326	}
11327	#endif
11328
11329
11330	/**
11331	* VM-exit exception handler for #PF (Page-fault exception).
11332	*/
11333	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11334	{
11335	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11336	PVM pVM = pVCpu->CTX_SUFF(pVM);
11337	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11338	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11339	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11340	AssertRCReturn(rc, rc);
11341
11342	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11343	if (pVM->hm.s.fNestedPaging)
11344	{
11345	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11346	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11347	{
11348	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11349	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11350	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11351	}
11352	else
11353	{
11354	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11355	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11356	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11357	}
11358	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11359	return rc;
11360	}
11361	#else
11362	Assert(!pVM->hm.s.fNestedPaging);
11363	NOREF(pVM);
11364	#endif
11365
11366	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11367	AssertRCReturn(rc, rc);
11368
11369	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11370	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11371
11372	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11373	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11374	(RTGCPTR)pVmxTransient->uExitQualification);
11375
11376	Log4(("#PF: rc=%Rrc\n", rc));
11377	if (rc == VINF_SUCCESS)
11378	{
11379	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11380	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11381	* memory? We don't update the whole state here... */
11382	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11383	\| HM_CHANGED_GUEST_RSP
11384	\| HM_CHANGED_GUEST_RFLAGS
11385	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11386	TRPMResetTrap(pVCpu);
11387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11388	return rc;
11389	}
11390	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11391	{
11392	if (!pVmxTransient->fVectoringPF)
11393	{
11394	/* It's a guest page fault and needs to be reflected to the guest. */
11395	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11396	TRPMResetTrap(pVCpu);
11397	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11398	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11399	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11400	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11401	}
11402	else
11403	{
11404	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11405	TRPMResetTrap(pVCpu);
11406	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11407	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11408	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11409	}
11410
11411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11412	return VINF_SUCCESS;
11413	}
11414
11415	TRPMResetTrap(pVCpu);
11416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11417	return rc;
11418	}
11419
11420	/** @} */
11421

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49966

Download in other formats: