HMSVMR0.cpp@ 72777

Last change on this file since 72777 was 72777, checked in by vboxsync, 6 years ago
HM: nit
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 318.5 KB

Line
1	/* $Id: HMSVMR0.cpp 72777 2018-06-29 18:04:40Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/gim.h>
33	#include <VBox/vmm/apic.h>
34	#include "HMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include "HMSVMR0.h"
37	#include "dtrace/VBoxVMM.h"
38
39	#ifdef DEBUG_ramshankar
40	# define HMSVM_SYNC_FULL_GUEST_STATE
41	# define HMSVM_SYNC_FULL_NESTED_GUEST_STATE
42	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
43	# define HMSVM_ALWAYS_TRAP_PF
44	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
45	#endif
46
47
48	/*********************************************************************************************************************************
49	* Defined Constants And Macros *
50	*********************************************************************************************************************************/
51	#ifdef VBOX_WITH_STATISTICS
52	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
53	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
54	if ((u64ExitCode) == SVM_EXIT_NPF) \
55	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
56	else \
57	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
58	} while (0)
59
60	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
61	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
62	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
63	if ((u64ExitCode) == SVM_EXIT_NPF) \
64	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
65	else \
66	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
67	} while (0)
68	# endif
69	#else
70	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
71	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
72	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
73	# endif
74	#endif /* !VBOX_WITH_STATISTICS */
75
76
77	/** If we decide to use a function table approach this can be useful to
78	* switch to a "static DECLCALLBACK(int)". */
79	#define HMSVM_EXIT_DECL static int
80
81	/**
82	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
83	* guest using hardware-assisted SVM.
84	*
85	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
86	* are swapped and restored across the world-switch and also registers like
87	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
88	* \#VMEXIT.
89	*/
90	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
91	\| CPUMCTX_EXTRN_RFLAGS \
92	\| CPUMCTX_EXTRN_RAX \
93	\| CPUMCTX_EXTRN_RSP \
94	\| CPUMCTX_EXTRN_SREG_MASK \
95	\| CPUMCTX_EXTRN_CR0 \
96	\| CPUMCTX_EXTRN_CR2 \
97	\| CPUMCTX_EXTRN_CR3 \
98	\| CPUMCTX_EXTRN_TABLE_MASK \
99	\| CPUMCTX_EXTRN_DR6 \
100	\| CPUMCTX_EXTRN_DR7 \
101	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
102	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
103	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
104	\| CPUMCTX_EXTRN_HWVIRT \
105	\| CPUMCTX_EXTRN_HM_SVM_MASK)
106
107	/**
108	* Subset of the guest-CPU state that is shared between the guest and host.
109	*/
110	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
111
112	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
113	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
114	do { \
115	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
116	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
117	} while (0)
118
119	/** Assert that the required state bits are fetched. */
120	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
121	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
122	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
123
124	/** Macro for checking and returning from the using function for
125	* \#VMEXIT intercepts that maybe caused during delivering of another
126	* event in the guest. */
127	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
128	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY() \
129	do \
130	{ \
131	int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
132	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
133	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
134	else if ( rc == VINF_EM_RESET \
135	&& CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
136	{ \
137	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK); \
138	return VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_SHUTDOWN, 0, 0)); \
139	} \
140	else \
141	return rc; \
142	} while (0)
143	#else
144	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY() \
145	do \
146	{ \
147	int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
148	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
149	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
150	else \
151	return rc; \
152	} while (0)
153	#endif
154
155	/** Macro which updates interrupt shadow for the current RIP. */
156	#define HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx) \
157	do { \
158	/* Update interrupt shadow. */ \
159	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS) \
160	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu)) \
161	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); \
162	} while (0)
163
164	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
165	* instruction that exited. */
166	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
167	do { \
168	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
169	(a_rc) = VINF_EM_DBG_STEPPED; \
170	} while (0)
171
172	/** Assert that preemption is disabled or covered by thread-context hooks. */
173	#define HMSVM_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
174	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
175
176	/** Assert that we haven't migrated CPUs when thread-context hooks are not
177	* used. */
178	#define HMSVM_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
180	("Illegal migration! Entered on CPU %u Current %u\n", \
181	pVCpu->hm.s.idEnteredCpu, RTMpCpuId()));
182
183	/** Assert that we're not executing a nested-guest. */
184	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
185	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
186	#else
187	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
188	#endif
189
190	/** Assert that we're executing a nested-guest. */
191	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
192	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
193	#else
194	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
195	#endif
196
197	/** Validate segment descriptor granularity bit. */
198	#ifdef VBOX_STRICT
199	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
200	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
201	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
202	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
203	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
204	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
205	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
206	#else
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
208	#endif
209
210	/**
211	* Exception bitmap mask for all contributory exceptions.
212	*
213	* Page fault is deliberately excluded here as it's conditional as to whether
214	* it's contributory or benign. Page faults are handled separately.
215	*/
216	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
217	\| RT_BIT(X86_XCPT_DE))
218
219	/**
220	* Mandatory/unconditional guest control intercepts.
221	*
222	* SMIs can and do happen in normal operation. We need not intercept them
223	* while executing the guest (or nested-guest).
224	*/
225	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
226	\| SVM_CTRL_INTERCEPT_NMI \
227	\| SVM_CTRL_INTERCEPT_INIT \
228	\| SVM_CTRL_INTERCEPT_RDPMC \
229	\| SVM_CTRL_INTERCEPT_CPUID \
230	\| SVM_CTRL_INTERCEPT_RSM \
231	\| SVM_CTRL_INTERCEPT_HLT \
232	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
233	\| SVM_CTRL_INTERCEPT_MSR_PROT \
234	\| SVM_CTRL_INTERCEPT_INVLPGA \
235	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
236	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
237	\| SVM_CTRL_INTERCEPT_VMRUN \
238	\| SVM_CTRL_INTERCEPT_SKINIT \
239	\| SVM_CTRL_INTERCEPT_WBINVD \
240	\| SVM_CTRL_INTERCEPT_MONITOR \
241	\| SVM_CTRL_INTERCEPT_MWAIT \
242	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
243	\| SVM_CTRL_INTERCEPT_XSETBV)
244
245	/** @name VMCB Clean Bits.
246	*
247	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
248	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
249	* memory.
250	*
251	* @{ */
252	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
253	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
254	/** I/O permission bitmap, MSR permission bitmap. */
255	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
256	/** ASID. */
257	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
258	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
259	V_INTR_VECTOR. */
260	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
261	/** Nested Paging: Nested CR3 (nCR3), PAT. */
262	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
263	/** Control registers (CR0, CR3, CR4, EFER). */
264	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
265	/** Debug registers (DR6, DR7). */
266	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
267	/** GDT, IDT limit and base. */
268	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
269	/** Segment register: CS, SS, DS, ES limit and base. */
270	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
271	/** CR2.*/
272	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
273	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
274	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
275	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
276	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
277	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
278	/** Mask of all valid VMCB Clean bits. */
279	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
280	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
281	\| HMSVM_VMCB_CLEAN_ASID \
282	\| HMSVM_VMCB_CLEAN_INT_CTRL \
283	\| HMSVM_VMCB_CLEAN_NP \
284	\| HMSVM_VMCB_CLEAN_CRX_EFER \
285	\| HMSVM_VMCB_CLEAN_DRX \
286	\| HMSVM_VMCB_CLEAN_DT \
287	\| HMSVM_VMCB_CLEAN_SEG \
288	\| HMSVM_VMCB_CLEAN_CR2 \
289	\| HMSVM_VMCB_CLEAN_LBR \
290	\| HMSVM_VMCB_CLEAN_AVIC)
291	/** @} */
292
293	/** @name SVM transient.
294	*
295	* A state structure for holding miscellaneous information across AMD-V
296	* VMRUN/\#VMEXIT operation, restored after the transition.
297	*
298	* @{ */
299	typedef struct SVMTRANSIENT
300	{
301	/** The host's rflags/eflags. */
302	RTCCUINTREG fEFlags;
303	#if HC_ARCH_BITS == 32
304	uint32_t u32Alignment0;
305	#endif
306
307	/** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
308	uint64_t u64ExitCode;
309	/** The guest's TPR value used for TPR shadowing. */
310	uint8_t u8GuestTpr;
311	/** Alignment. */
312	uint8_t abAlignment0[7];
313
314	/** Pointer to the currently executing VMCB. */
315	PSVMVMCB pVmcb;
316	/** Whether we are currently executing a nested-guest. */
317	bool fIsNestedGuest;
318
319	/** Whether the guest debug state was active at the time of \#VMEXIT. */
320	bool fWasGuestDebugStateActive;
321	/** Whether the hyper debug state was active at the time of \#VMEXIT. */
322	bool fWasHyperDebugStateActive;
323	/** Whether the TSC offset mode needs to be updated. */
324	bool fUpdateTscOffsetting;
325	/** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
326	bool fRestoreTscAuxMsr;
327	/** Whether the \#VMEXIT was caused by a page-fault during delivery of a
328	* contributary exception or a page-fault. */
329	bool fVectoringDoublePF;
330	/** Whether the \#VMEXIT was caused by a page-fault during delivery of an
331	* external interrupt or NMI. */
332	bool fVectoringPF;
333	} SVMTRANSIENT, *PSVMTRANSIENT;
334	AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
335	AssertCompileMemberAlignment(SVMTRANSIENT, pVmcb, sizeof(uint64_t));
336	/** @} */
337
338	/**
339	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
340	*/
341	typedef enum SVMMSREXITREAD
342	{
343	/** Reading this MSR causes a \#VMEXIT. */
344	SVMMSREXIT_INTERCEPT_READ = 0xb,
345	/** Reading this MSR does not cause a \#VMEXIT. */
346	SVMMSREXIT_PASSTHRU_READ
347	} SVMMSREXITREAD;
348
349	/**
350	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
351	*/
352	typedef enum SVMMSREXITWRITE
353	{
354	/** Writing to this MSR causes a \#VMEXIT. */
355	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
356	/** Writing to this MSR does not cause a \#VMEXIT. */
357	SVMMSREXIT_PASSTHRU_WRITE
358	} SVMMSREXITWRITE;
359
360	/**
361	* SVM \#VMEXIT handler.
362	*
363	* @returns VBox status code.
364	* @param pVCpu The cross context virtual CPU structure.
365	* @param pCtx Pointer to the guest-CPU context.
366	* @param pSvmTransient Pointer to the SVM-transient structure.
367	*/
368	typedef int FNSVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
369
370
371	/*********************************************************************************************************************************
372	* Internal Functions *
373	*********************************************************************************************************************************/
374	static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu);
375	static void hmR0SvmLeave(PVMCPU pVCpu, bool fImportState);
376
377
378	/** @name \#VMEXIT handlers.
379	* @{
380	*/
381	static FNSVMEXITHANDLER hmR0SvmExitIntr;
382	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
383	static FNSVMEXITHANDLER hmR0SvmExitInvd;
384	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
385	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
386	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
387	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
388	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
389	static FNSVMEXITHANDLER hmR0SvmExitHlt;
390	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
391	static FNSVMEXITHANDLER hmR0SvmExitMwait;
392	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
393	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
394	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
395	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
396	static FNSVMEXITHANDLER hmR0SvmExitMsr;
397	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
398	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
399	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
400	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
401	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
402	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
403	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
404	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
405	static FNSVMEXITHANDLER hmR0SvmExitPause;
406	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
407	static FNSVMEXITHANDLER hmR0SvmExitIret;
408	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
409	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
410	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
411	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
412	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
413	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
414	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM)
415	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
416	#endif
417	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
418	static FNSVMEXITHANDLER hmR0SvmExitClgi;
419	static FNSVMEXITHANDLER hmR0SvmExitStgi;
420	static FNSVMEXITHANDLER hmR0SvmExitVmload;
421	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
422	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
423	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
424	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
425	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
426	#endif
427	/** @} */
428
429	static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
430	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
431	static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
432	#endif
433
434
435	/*********************************************************************************************************************************
436	* Global Variables *
437	*********************************************************************************************************************************/
438	/** Ring-0 memory object for the IO bitmap. */
439	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
440	/** Physical address of the IO bitmap. */
441	static RTHCPHYS g_HCPhysIOBitmap;
442	/** Pointer to the IO bitmap. */
443	static R0PTRTYPE(void *) g_pvIOBitmap;
444
445	#ifdef VBOX_STRICT
446	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
447	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
448	# define HMSVM_LOG_CS RT_BIT_32(2)
449	# define HMSVM_LOG_SS RT_BIT_32(3)
450	# define HMSVM_LOG_FS RT_BIT_32(4)
451	# define HMSVM_LOG_GS RT_BIT_32(5)
452	# define HMSVM_LOG_LBR RT_BIT_32(6)
453	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
454	\| HMSVM_LOG_CR_REGS \
455	\| HMSVM_LOG_CS \
456	\| HMSVM_LOG_SS \
457	\| HMSVM_LOG_FS \
458	\| HMSVM_LOG_GS \
459	\| HMSVM_LOG_LBR)
460
461	/**
462	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
463	*
464	* @param pVCpu The cross context virtual CPU structure.
465	* @param pVmcb Pointer to the VM control block.
466	* @param pszPrefix Log prefix.
467	* @param fFlags Log flags, see HMSVM_LOG_XXX.
468	* @param uVerbose The verbosity level, currently unused.
469	*/
470	static void hmR0SvmLogState(PVMCPU pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
471	{
472	RT_NOREF2(pVCpu, uVerbose);
473	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
474
475	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
476	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
477
478	if (fFlags & HMSVM_LOG_RBP_RSP)
479	{
480	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
481	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
482	}
483
484	if (fFlags & HMSVM_LOG_CR_REGS)
485	{
486	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
487	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
488	}
489
490	if (fFlags & HMSVM_LOG_CS)
491	{
492	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
493	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
494	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
495	}
496	if (fFlags & HMSVM_LOG_SS)
497	{
498	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
499	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
500	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
501	}
502	if (fFlags & HMSVM_LOG_FS)
503	{
504	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
505	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
506	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
507	}
508	if (fFlags & HMSVM_LOG_GS)
509	{
510	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
511	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
512	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
513	}
514
515	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
516	if (fFlags & HMSVM_LOG_LBR)
517	{
518	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
519	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
520	}
521	NOREF(pVmcbGuest); NOREF(pCtx);
522	}
523	#endif /* VBOX_STRICT */
524
525
526	/**
527	* Sets up and activates AMD-V on the current CPU.
528	*
529	* @returns VBox status code.
530	* @param pCpu Pointer to the CPU info struct.
531	* @param pVM The cross context VM structure. Can be
532	* NULL after a resume!
533	* @param pvCpuPage Pointer to the global CPU page.
534	* @param HCPhysCpuPage Physical address of the global CPU page.
535	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
536	* @param pvArg Unused on AMD-V.
537	*/
538	VMMR0DECL(int) SVMR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
539	void *pvArg)
540	{
541	Assert(!fEnabledByHost);
542	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
543	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
544	Assert(pvCpuPage); NOREF(pvCpuPage);
545	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
546
547	NOREF(pvArg);
548	NOREF(fEnabledByHost);
549
550	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
551	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
552
553	/*
554	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
555	*/
556	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
557	if (u64HostEfer & MSR_K6_EFER_SVME)
558	{
559	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
560	if ( pVM
561	&& pVM->hm.s.svm.fIgnoreInUseError)
562	pCpu->fIgnoreAMDVInUseError = true;
563
564	if (!pCpu->fIgnoreAMDVInUseError)
565	{
566	ASMSetFlags(fEFlags);
567	return VERR_SVM_IN_USE;
568	}
569	}
570
571	/* Turn on AMD-V in the EFER MSR. */
572	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
573
574	/* Write the physical page address where the CPU will store the host state while executing the VM. */
575	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
576
577	/* Restore interrupts. */
578	ASMSetFlags(fEFlags);
579
580	/*
581	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
582	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
583	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
584	* entirely with before executing any guest code.
585	*/
586	pCpu->fFlushAsidBeforeUse = true;
587
588	/*
589	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
590	*/
591	++pCpu->cTlbFlushes;
592
593	return VINF_SUCCESS;
594	}
595
596
597	/**
598	* Deactivates AMD-V on the current CPU.
599	*
600	* @returns VBox status code.
601	* @param pCpu Pointer to the CPU info struct.
602	* @param pvCpuPage Pointer to the global CPU page.
603	* @param HCPhysCpuPage Physical address of the global CPU page.
604	*/
605	VMMR0DECL(int) SVMR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
606	{
607	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
608	AssertReturn( HCPhysCpuPage
609	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
610	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
611	NOREF(pCpu);
612
613	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
614	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
615
616	/* Turn off AMD-V in the EFER MSR. */
617	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
618	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
619
620	/* Invalidate host state physical address. */
621	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
622
623	/* Restore interrupts. */
624	ASMSetFlags(fEFlags);
625
626	return VINF_SUCCESS;
627	}
628
629
630	/**
631	* Does global AMD-V initialization (called during module initialization).
632	*
633	* @returns VBox status code.
634	*/
635	VMMR0DECL(int) SVMR0GlobalInit(void)
636	{
637	/*
638	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
639	* intercept all IO accesses, it's done once globally here instead of per-VM.
640	*/
641	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
642	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
643	if (RT_FAILURE(rc))
644	return rc;
645
646	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
647	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
648
649	/* Set all bits to intercept all IO accesses. */
650	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
651
652	return VINF_SUCCESS;
653	}
654
655
656	/**
657	* Does global AMD-V termination (called during module termination).
658	*/
659	VMMR0DECL(void) SVMR0GlobalTerm(void)
660	{
661	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
662	{
663	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
664	g_pvIOBitmap = NULL;
665	g_HCPhysIOBitmap = 0;
666	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
667	}
668	}
669
670
671	/**
672	* Frees any allocated per-VCPU structures for a VM.
673	*
674	* @param pVM The cross context VM structure.
675	*/
676	DECLINLINE(void) hmR0SvmFreeStructs(PVM pVM)
677	{
678	for (uint32_t i = 0; i < pVM->cCpus; i++)
679	{
680	PVMCPU pVCpu = &pVM->aCpus[i];
681	AssertPtr(pVCpu);
682
683	if (pVCpu->hm.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
684	{
685	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcbHost, false);
686	pVCpu->hm.s.svm.HCPhysVmcbHost = 0;
687	pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
688	}
689
690	if (pVCpu->hm.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
691	{
692	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcb, false);
693	pVCpu->hm.s.svm.pVmcb = NULL;
694	pVCpu->hm.s.svm.HCPhysVmcb = 0;
695	pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
696	}
697
698	if (pVCpu->hm.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
699	{
700	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjMsrBitmap, false);
701	pVCpu->hm.s.svm.pvMsrBitmap = NULL;
702	pVCpu->hm.s.svm.HCPhysMsrBitmap = 0;
703	pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
704	}
705	}
706	}
707
708
709	/**
710	* Does per-VM AMD-V initialization.
711	*
712	* @returns VBox status code.
713	* @param pVM The cross context VM structure.
714	*/
715	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
716	{
717	int rc = VERR_INTERNAL_ERROR_5;
718
719	/*
720	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
721	*/
722	uint32_t u32Family;
723	uint32_t u32Model;
724	uint32_t u32Stepping;
725	if (HMAmdIsSubjectToErratum170(&u32Family, &u32Model, &u32Stepping))
726	{
727	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
728	pVM->hm.s.svm.fAlwaysFlushTLB = true;
729	}
730
731	/*
732	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
733	*/
734	for (VMCPUID i = 0; i < pVM->cCpus; i++)
735	{
736	PVMCPU pVCpu = &pVM->aCpus[i];
737	pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
738	pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
739	pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
740	}
741
742	for (VMCPUID i = 0; i < pVM->cCpus; i++)
743	{
744	PVMCPU pVCpu = &pVM->aCpus[i];
745
746	/*
747	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
748	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
749	*/
750	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
751	if (RT_FAILURE(rc))
752	goto failure_cleanup;
753
754	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcbHost);
755	pVCpu->hm.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcbHost, 0 /* iPage */);
756	Assert(pVCpu->hm.s.svm.HCPhysVmcbHost < _4G);
757	ASMMemZeroPage(pvVmcbHost);
758
759	/*
760	* Allocate one page for the guest-state VMCB.
761	*/
762	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
763	if (RT_FAILURE(rc))
764	goto failure_cleanup;
765
766	pVCpu->hm.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcb);
767	pVCpu->hm.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcb, 0 /* iPage */);
768	Assert(pVCpu->hm.s.svm.HCPhysVmcb < _4G);
769	ASMMemZeroPage(pVCpu->hm.s.svm.pVmcb);
770
771	/*
772	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
773	* SVM to not require one.
774	*/
775	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
776	false /* fExecutable */);
777	if (RT_FAILURE(rc))
778	goto failure_cleanup;
779
780	pVCpu->hm.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjMsrBitmap);
781	pVCpu->hm.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
782	/* Set all bits to intercept all MSR accesses (changed later on). */
783	ASMMemFill32(pVCpu->hm.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
784	}
785
786	return VINF_SUCCESS;
787
788	failure_cleanup:
789	hmR0SvmFreeStructs(pVM);
790	return rc;
791	}
792
793
794	/**
795	* Does per-VM AMD-V termination.
796	*
797	* @returns VBox status code.
798	* @param pVM The cross context VM structure.
799	*/
800	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
801	{
802	hmR0SvmFreeStructs(pVM);
803	return VINF_SUCCESS;
804	}
805
806
807	/**
808	* Returns whether the VMCB Clean Bits feature is supported.
809	*
810	* @return @c true if supported, @c false otherwise.
811	* @param pVCpu The cross context virtual CPU structure.
812	* @param pCtx Pointer to the guest-CPU context.
813	*/
814	DECLINLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPU pVCpu, PCCPUMCTX pCtx)
815	{
816	PVM pVM = pVCpu->CTX_SUFF(pVM);
817	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
818	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
819	{
820	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN)
821	&& pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
822	}
823	#else
824	RT_NOREF(pCtx);
825	#endif
826	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
827	}
828
829
830	/**
831	* Returns whether the decode assists feature is supported.
832	*
833	* @return @c true if supported, @c false otherwise.
834	* @param pVCpu The cross context virtual CPU structure.
835	* @param pCtx Pointer to the guest-CPU context.
836	*/
837	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPU pVCpu, PCPUMCTX pCtx)
838	{
839	PVM pVM = pVCpu->CTX_SUFF(pVM);
840	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
841	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
842	{
843	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
844	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
845	}
846	#else
847	RT_NOREF(pCtx);
848	#endif
849	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
850	}
851
852
853	/**
854	* Returns whether the NRIP_SAVE feature is supported.
855	*
856	* @return @c true if supported, @c false otherwise.
857	* @param pVCpu The cross context virtual CPU structure.
858	* @param pCtx Pointer to the guest-CPU context.
859	*/
860	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPU pVCpu, PCPUMCTX pCtx)
861	{
862	PVM pVM = pVCpu->CTX_SUFF(pVM);
863	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
864	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
865	{
866	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
867	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
868	}
869	#else
870	RT_NOREF(pCtx);
871	#endif
872	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
873	}
874
875
876	/**
877	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
878	*
879	* @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
880	* @param pbMsrBitmap Pointer to the MSR bitmap.
881	* @param idMsr The MSR for which the permissions are being set.
882	* @param enmRead MSR read permissions.
883	* @param enmWrite MSR write permissions.
884	*
885	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
886	* caller needs to take care of this.
887	*/
888	static void hmR0SvmSetMsrPermission(PCCPUMCTX pCtx, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
889	SVMMSREXITWRITE enmWrite)
890	{
891	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
892	uint16_t offMsrpm;
893	uint8_t uMsrpmBit;
894	int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
895	AssertRC(rc);
896
897	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
898	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
899
900	pbMsrBitmap += offMsrpm;
901	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
902	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
903	else
904	{
905	if (!fInNestedGuestMode)
906	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
907	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
908	else
909	{
910	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
911	uint8_t const pbNstGstMsrBitmap = (uint8_t )pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
912	pbNstGstMsrBitmap += offMsrpm;
913	if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit)))
914	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
915	else
916	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
917	}
918	#endif
919	}
920
921	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
922	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
923	else
924	{
925	if (!fInNestedGuestMode)
926	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
927	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
928	else
929	{
930	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
931	uint8_t const pbNstGstMsrBitmap = (uint8_t )pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
932	pbNstGstMsrBitmap += offMsrpm;
933	if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit + 1)))
934	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
935	else
936	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
937	}
938	#endif
939	}
940	}
941
942
943	/**
944	* Sets up AMD-V for the specified VM.
945	* This function is only called once per-VM during initalization.
946	*
947	* @returns VBox status code.
948	* @param pVM The cross context VM structure.
949	*/
950	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
951	{
952	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
953	AssertReturn(pVM, VERR_INVALID_PARAMETER);
954	Assert(pVM->hm.s.svm.fSupported);
955
956	bool const fPauseFilter = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
957	bool const fPauseFilterThreshold = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
958	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
959
960	bool const fLbrVirt = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
961	bool const fUseLbrVirt = fLbrVirt; /** @todo CFGM, IEM implementation etc. */
962
963	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
964	bool const fVirtVmsaveVmload = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
965	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && pVM->hm.s.fNestedPaging;
966
967	bool const fVGif = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);
968	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
969	#endif
970
971	PVMCPU pVCpu = &pVM->aCpus[0];
972	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
973	AssertMsgReturn(pVmcb, ("Invalid pVmcb for vcpu[0]\n"), VERR_SVM_INVALID_PVMCB);
974	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
975
976	/* Always trap #AC for reasons of security. */
977	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
978
979	/* Always trap #DB for reasons of security. */
980	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
981
982	/* Trap exceptions unconditionally (debug purposes). */
983	#ifdef HMSVM_ALWAYS_TRAP_PF
984	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
985	#endif
986	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
987	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
988	pVmcbCtrl->u32InterceptXcpt \|= 0
989	\| RT_BIT(X86_XCPT_BP)
990	\| RT_BIT(X86_XCPT_DE)
991	\| RT_BIT(X86_XCPT_NM)
992	\| RT_BIT(X86_XCPT_UD)
993	\| RT_BIT(X86_XCPT_NP)
994	\| RT_BIT(X86_XCPT_SS)
995	\| RT_BIT(X86_XCPT_GP)
996	\| RT_BIT(X86_XCPT_PF)
997	\| RT_BIT(X86_XCPT_MF)
998	;
999	#endif
1000
1001	/* Apply the exceptions intercepts needed by the GIM provider. */
1002	if (pVCpu->hm.s.fGIMTrapXcptUD)
1003	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1004
1005	/* Set up unconditional intercepts and conditions. */
1006	pVmcbCtrl->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1007	\| SVM_CTRL_INTERCEPT_VMMCALL;
1008
1009	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1010	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1011	#endif
1012
1013	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1014	/* Virtualized VMSAVE/VMLOAD. */
1015	pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload = fUseVirtVmsaveVmload;
1016	if (!fUseVirtVmsaveVmload)
1017	{
1018	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
1019	\| SVM_CTRL_INTERCEPT_VMLOAD;
1020	}
1021
1022	/* Virtual GIF. */
1023	pVmcbCtrl->IntCtrl.n.u1VGifEnable = fUseVGif;
1024	if (!fUseVGif)
1025	{
1026	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
1027	\| SVM_CTRL_INTERCEPT_STGI;
1028	}
1029	#endif
1030
1031	/* CR4 writes must always be intercepted for tracking PGM mode changes. */
1032	pVmcbCtrl->u16InterceptWrCRx = RT_BIT(4);
1033
1034	/* Intercept all DRx reads and writes by default. Changed later on. */
1035	pVmcbCtrl->u16InterceptRdDRx = 0xffff;
1036	pVmcbCtrl->u16InterceptWrDRx = 0xffff;
1037
1038	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1039	pVmcbCtrl->IntCtrl.n.u1VIntrMasking = 1;
1040
1041	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1042	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1043	pVmcbCtrl->IntCtrl.n.u1IgnoreTPR = 1;
1044
1045	/* Set the IO permission bitmap physical addresses. */
1046	pVmcbCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1047
1048	/* LBR virtualization. */
1049	pVmcbCtrl->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1050
1051	/* The host ASID MBZ, for the guest start with 1. */
1052	pVmcbCtrl->TLBCtrl.n.u32ASID = 1;
1053
1054	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1055	pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging = pVM->hm.s.fNestedPaging;
1056
1057	/* Without Nested Paging, we need additionally intercepts. */
1058	if (!pVM->hm.s.fNestedPaging)
1059	{
1060	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1061	pVmcbCtrl->u16InterceptRdCRx \|= RT_BIT(3);
1062	pVmcbCtrl->u16InterceptWrCRx \|= RT_BIT(3);
1063
1064	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1065	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1066	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1067
1068	/* Page faults must be intercepted to implement shadow paging. */
1069	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1070	}
1071
1072	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1073	if (fUsePauseFilter)
1074	{
1075	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1076	pVmcbCtrl->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1077	if (fPauseFilterThreshold)
1078	pVmcbCtrl->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1079	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1080	}
1081
1082	/*
1083	* Setup the MSR permission bitmap.
1084	* The following MSRs are saved/restored automatically during the world-switch.
1085	* Don't intercept guest read/write accesses to these MSRs.
1086	*/
1087	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
1088	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1089	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1090	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1091	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1092	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1093	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1094	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1095	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1096	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1097	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1098	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1099	pVmcbCtrl->u64MSRPMPhysAddr = pVCpu->hm.s.svm.HCPhysMsrBitmap;
1100
1101	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1102	Assert(pVmcbCtrl->u32VmcbCleanBits == 0);
1103
1104	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1105	{
1106	PVMCPU pVCpuCur = &pVM->aCpus[i];
1107	PSVMVMCB pVmcbCur = pVM->aCpus[i].hm.s.svm.pVmcb;
1108	AssertMsgReturn(pVmcbCur, ("Invalid pVmcb for vcpu[%u]\n", i), VERR_SVM_INVALID_PVMCB);
1109	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1110
1111	/* Copy the VMCB control area. */
1112	memcpy(pVmcbCtrlCur, pVmcbCtrl, sizeof(*pVmcbCtrlCur));
1113
1114	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1115	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hm.s.svm.pvMsrBitmap;
1116	memcpy(pbMsrBitmapCur, pbMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1117	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hm.s.svm.HCPhysMsrBitmap;
1118
1119	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1120	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1121
1122	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1123	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu->hm.s.fGIMTrapXcptUD);
1124	}
1125
1126	return VINF_SUCCESS;
1127	}
1128
1129
1130	/**
1131	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1132	*
1133	* @returns Pointer to the current context VMCB.
1134	* @param pVCpu The cross context virtual CPU structure.
1135	* @param pCtx Pointer to the guest-CPU context.
1136	*/
1137	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
1138	{
1139	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1140	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1141	return pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
1142	#else
1143	RT_NOREF(pCtx);
1144	#endif
1145	return pVCpu->hm.s.svm.pVmcb;
1146	}
1147
1148
1149	/**
1150	* Gets a pointer to the nested-guest VMCB cache.
1151	*
1152	* @returns Pointer to the nested-guest VMCB cache.
1153	* @param pVCpu The cross context virtual CPU structure.
1154	*/
1155	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPU pVCpu)
1156	{
1157	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1158	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1159	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1160	#else
1161	RT_NOREF(pVCpu);
1162	return NULL;
1163	#endif
1164	}
1165
1166
1167	/**
1168	* Invalidates a guest page by guest virtual address.
1169	*
1170	* @returns VBox status code.
1171	* @param pVM The cross context VM structure.
1172	* @param pVCpu The cross context virtual CPU structure.
1173	* @param GCVirt Guest virtual address of the page to invalidate.
1174	*/
1175	VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1176	{
1177	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1178	Assert(pVM->hm.s.svm.fSupported);
1179
1180	bool fFlushPending = pVM->hm.s.svm.fAlwaysFlushTLB \|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1181
1182	/* Skip it if a TLB flush is already pending. */
1183	if (!fFlushPending)
1184	{
1185	Log4Func(("%#RGv\n", GCVirt));
1186
1187	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1188	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
1189	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1190
1191	#if HC_ARCH_BITS == 32
1192	/* If we get a flush in 64-bit guest mode, then force a full TLB flush. INVLPGA takes only 32-bit addresses. */
1193	if (CPUMIsGuestInLongMode(pVCpu))
1194	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1195	else
1196	#endif
1197	{
1198	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1199	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1200	}
1201	}
1202	return VINF_SUCCESS;
1203	}
1204
1205
1206	/**
1207	* Flushes the appropriate tagged-TLB entries.
1208	*
1209	* @param pVCpu The cross context virtual CPU structure.
1210	* @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
1211	* @param pVmcb Pointer to the VM control block.
1212	* @param pHostCpu Pointer to the HM host-CPU info.
1213	*/
1214	static void hmR0SvmFlushTaggedTlb(PVMCPU pVCpu, PCCPUMCTX pCtx, PSVMVMCB pVmcb, PHMGLOBALCPUINFO pHostCpu)
1215	{
1216	#ifndef VBOX_WITH_NESTED_HWVIRT_SVM
1217	RT_NOREF(pCtx);
1218	#endif
1219	PVM pVM = pVCpu->CTX_SUFF(pVM);
1220
1221	/*
1222	* Force a TLB flush for the first world switch if the current CPU differs from the one
1223	* we ran on last. This can happen both for start & resume due to long jumps back to
1224	* ring-3.
1225	*
1226	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1227	* correlation between it and the physical CPU.
1228	*
1229	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1230	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1231	*/
1232	bool fNewAsid = false;
1233	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1234	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
1235	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes
1236	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1237	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
1238	#endif
1239	)
1240	{
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1242	pVCpu->hm.s.fForceTLBFlush = true;
1243	fNewAsid = true;
1244	}
1245
1246	/* Set TLB flush state as checked until we return from the world switch. */
1247	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1248
1249	/* Check for explicit TLB flushes. */
1250	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1251	{
1252	pVCpu->hm.s.fForceTLBFlush = true;
1253	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1254	}
1255
1256	/*
1257	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1258	* This Host CPU requirement takes precedence.
1259	*/
1260	if (pVM->hm.s.svm.fAlwaysFlushTLB)
1261	{
1262	pHostCpu->uCurrentAsid = 1;
1263	pVCpu->hm.s.uCurrentAsid = 1;
1264	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1265	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
1266	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1267
1268	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1269	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1270	}
1271	else
1272	{
1273	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1274	if (pVCpu->hm.s.fForceTLBFlush)
1275	{
1276	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1277	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1278
1279	if (fNewAsid)
1280	{
1281	++pHostCpu->uCurrentAsid;
1282
1283	bool fHitASIDLimit = false;
1284	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1285	{
1286	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1287	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1288	fHitASIDLimit = true;
1289	}
1290
1291	if ( fHitASIDLimit
1292	\|\| pHostCpu->fFlushAsidBeforeUse)
1293	{
1294	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1295	pHostCpu->fFlushAsidBeforeUse = false;
1296	}
1297
1298	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1299	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
1300	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1301	}
1302	else
1303	{
1304	if (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1305	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1306	else
1307	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1308	}
1309
1310	pVCpu->hm.s.fForceTLBFlush = false;
1311	}
1312	}
1313
1314	/* Update VMCB with the ASID. */
1315	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hm.s.uCurrentAsid)
1316	{
1317	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentAsid;
1318	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1319	}
1320
1321	AssertMsg(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu,
1322	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hm.s.idLastCpu, pHostCpu->idCpu));
1323	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1324	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1325	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1326	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1327	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1328	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1329
1330	#ifdef VBOX_WITH_STATISTICS
1331	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1332	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1333	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1334	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1335	{
1336	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1337	}
1338	else
1339	{
1340	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1341	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1342	}
1343	#endif
1344	}
1345
1346
1347	/** @name 64-bit guest on 32-bit host OS helper functions.
1348	*
1349	* The host CPU is still 64-bit capable but the host OS is running in 32-bit
1350	* mode (code segment, paging). These wrappers/helpers perform the necessary
1351	* bits for the 32->64 switcher.
1352	*
1353	* @{ */
1354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1355	/**
1356	* Prepares for and executes VMRUN (64-bit guests on a 32-bit host).
1357	*
1358	* @returns VBox status code.
1359	* @param HCPhysVmcbHost Physical address of host VMCB.
1360	* @param HCPhysVmcb Physical address of the VMCB.
1361	* @param pCtx Pointer to the guest-CPU context.
1362	* @param pVM The cross context VM structure.
1363	* @param pVCpu The cross context virtual CPU structure.
1364	*/
1365	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
1366	{
1367	RT_NOREF2(pVM, pCtx);
1368	uint32_t aParam[8];
1369	aParam[0] = RT_LO_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Lo. */
1370	aParam[1] = RT_HI_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Hi. */
1371	aParam[2] = RT_LO_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Lo. */
1372	aParam[3] = RT_HI_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Hi. */
1373	aParam[4] = VM_RC_ADDR(pVM, pVM);
1374	aParam[5] = 0;
1375	aParam[6] = VM_RC_ADDR(pVM, pVCpu);
1376	aParam[7] = 0;
1377
1378	return SVMR0Execute64BitsHandler(pVCpu, HM64ON32OP_SVMRCVMRun64, RT_ELEMENTS(aParam), &aParam[0]);
1379	}
1380
1381
1382	/**
1383	* Executes the specified VMRUN handler in 64-bit mode.
1384	*
1385	* @returns VBox status code.
1386	* @param pVCpu The cross context virtual CPU structure.
1387	* @param enmOp The operation to perform.
1388	* @param cParams Number of parameters.
1389	* @param paParam Array of 32-bit parameters.
1390	*/
1391	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
1392	{
1393	PVM pVM = pVCpu->CTX_SUFF(pVM);
1394	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
1395	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
1396
1397	/* Disable interrupts. */
1398	RTHCUINTREG const fEFlags = ASMIntDisableFlags();
1399
1400	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1401	RTCPUID idHostCpu = RTMpCpuId();
1402	CPUMR0SetLApic(pVCpu, idHostCpu);
1403	#endif
1404
1405	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
1406	CPUMSetHyperEIP(pVCpu, enmOp);
1407	for (int i = (int)cParams - 1; i >= 0; i--)
1408	CPUMPushHyper(pVCpu, paParam[i]);
1409
1410	/* Call the switcher. */
1411	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
1412	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
1413	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
1414
1415	/* Restore interrupts. */
1416	ASMSetFlags(fEFlags);
1417	return rc;
1418	}
1419
1420	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
1421	/** @} */
1422
1423
1424	/**
1425	* Sets an exception intercept in the specified VMCB.
1426	*
1427	* @param pVmcb Pointer to the VM control block.
1428	* @param uXcpt The exception (X86_XCPT_*).
1429	*/
1430	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1431	{
1432	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1433	{
1434	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1435	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1436	}
1437	}
1438
1439
1440	/**
1441	* Clears an exception intercept in the specified VMCB.
1442	*
1443	* @param pVCpu The cross context virtual CPU structure.
1444	* @param pVmcb Pointer to the VM control block.
1445	* @param uXcpt The exception (X86_XCPT_*).
1446	*
1447	* @remarks This takes into account if we're executing a nested-guest and only
1448	* removes the exception intercept if both the guest -and- nested-guest
1449	* are not intercepting it.
1450	*/
1451	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPU pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1452	{
1453	Assert(uXcpt != X86_XCPT_DB);
1454	Assert(uXcpt != X86_XCPT_AC);
1455	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1456	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1457	{
1458	bool fRemove = true;
1459	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1460	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1461	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1462	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1463	{
1464	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1465	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1466	}
1467	# else
1468	RT_NOREF(pVCpu);
1469	# endif
1470	if (fRemove)
1471	{
1472	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1473	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1474	}
1475	}
1476	#else
1477	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1478	#endif
1479	}
1480
1481
1482	/**
1483	* Sets a control intercept in the specified VMCB.
1484	*
1485	* @param pVmcb Pointer to the VM control block.
1486	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1487	*/
1488	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1489	{
1490	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1491	{
1492	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1493	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1494	}
1495	}
1496
1497
1498	/**
1499	* Clears a control intercept in the specified VMCB.
1500	*
1501	* @returns @c true if the intercept is still set, @c false otherwise.
1502	* @param pVCpu The cross context virtual CPU structure.
1503	* @param pCtx Pointer to the guest-CPU context.
1504	* @param pVmcb Pointer to the VM control block.
1505	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1506	*
1507	* @remarks This takes into account if we're executing a nested-guest and only
1508	* removes the control intercept if both the guest -and- nested-guest
1509	* are not intercepting it.
1510	*/
1511	DECLINLINE(bool) hmR0SvmClearCtrlIntercept(PVMCPU pVCpu, PCCPUMCTX pCtx, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1512	{
1513	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1514	{
1515	bool fRemove = true;
1516	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1517	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1518	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1519	{
1520	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1521	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1522	}
1523	#else
1524	RT_NOREF2(pVCpu, pCtx);
1525	#endif
1526	if (fRemove)
1527	{
1528	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1529	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1530	}
1531	}
1532
1533	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1534	}
1535
1536
1537	/**
1538	* Exports the guest (or nested-guest) CR0 into the VMCB.
1539	*
1540	* @param pVCpu The cross context virtual CPU structure.
1541	* @param pVmcb Pointer to the VM control block.
1542	*
1543	* @remarks This assumes we always pre-load the guest FPU.
1544	* @remarks No-long-jump zone!!!
1545	*/
1546	static void hmR0SvmExportGuestCR0(PVMCPU pVCpu, PSVMVMCB pVmcb)
1547	{
1548	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1549
1550	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1551	uint64_t const uGuestCr0 = pCtx->cr0;
1552	uint64_t uShadowCr0 = uGuestCr0;
1553
1554	/* Always enable caching. */
1555	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1556
1557	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1558	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1559	{
1560	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1561	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1562	}
1563
1564	/*
1565	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1566	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1567	* see @bugref{7243#c103}.
1568	*/
1569	if (!(uGuestCr0 & X86_CR0_NE))
1570	{
1571	uShadowCr0 \|= X86_CR0_NE;
1572	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1573	}
1574	else
1575	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1576
1577	/*
1578	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1579	*
1580	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1581	* see @bugref{6944}.
1582	*
1583	* We also don't ever want to honor weird things like cache disable from the guest.
1584	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1585	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1586	*/
1587	if (uShadowCr0 == uGuestCr0)
1588	{
1589	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1590	{
1591	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1592	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1593	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1594	}
1595	else
1596	{
1597	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1598	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1599	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1600	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1601	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1602	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1603	}
1604	}
1605	else
1606	{
1607	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1608	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1609	}
1610	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1611
1612	Assert(RT_HI_U32(uShadowCr0) == 0);
1613	if (pVmcb->guest.u64CR0 != uShadowCr0)
1614	{
1615	pVmcb->guest.u64CR0 = uShadowCr0;
1616	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1617	}
1618	}
1619
1620
1621	/**
1622	* Exports the guest (or nested-guest) CR3 into the VMCB.
1623	*
1624	* @param pVCpu The cross context virtual CPU structure.
1625	* @param pVmcb Pointer to the VM control block.
1626	*
1627	* @remarks No-long-jump zone!!!
1628	*/
1629	static void hmR0SvmExportGuestCR3(PVMCPU pVCpu, PSVMVMCB pVmcb)
1630	{
1631	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1632
1633	PVM pVM = pVCpu->CTX_SUFF(pVM);
1634	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1635	if (pVM->hm.s.fNestedPaging)
1636	{
1637	PGMMODE enmShwPagingMode;
1638	#if HC_ARCH_BITS == 32
1639	if (CPUMIsGuestInLongModeEx(pCtx))
1640	enmShwPagingMode = PGMMODE_AMD64_NX;
1641	else
1642	#endif
1643	enmShwPagingMode = PGMGetHostMode(pVM);
1644
1645	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
1646	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1647	pVmcb->guest.u64CR3 = pCtx->cr3;
1648	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1649	}
1650	else
1651	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1652
1653	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1654	}
1655
1656
1657	/**
1658	* Exports the guest (or nested-guest) CR4 into the VMCB.
1659	*
1660	* @param pVCpu The cross context virtual CPU structure.
1661	* @param pVmcb Pointer to the VM control block.
1662	*
1663	* @remarks No-long-jump zone!!!
1664	*/
1665	static int hmR0SvmExportGuestCR4(PVMCPU pVCpu, PSVMVMCB pVmcb)
1666	{
1667	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1668
1669	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1670	uint64_t uShadowCr4 = pCtx->cr4;
1671	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1672	{
1673	switch (pVCpu->hm.s.enmShadowMode)
1674	{
1675	case PGMMODE_REAL:
1676	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1677	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1678
1679	case PGMMODE_32_BIT: /* 32-bit paging. */
1680	uShadowCr4 &= ~X86_CR4_PAE;
1681	break;
1682
1683	case PGMMODE_PAE: /* PAE paging. */
1684	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1685	/** Must use PAE paging as we could use physical memory > 4 GB */
1686	uShadowCr4 \|= X86_CR4_PAE;
1687	break;
1688
1689	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1690	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1691	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1692	break;
1693	#else
1694	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1695	#endif
1696
1697	default: /* shut up gcc */
1698	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1699	}
1700	}
1701
1702	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1703	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1704
1705	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1706	if (uShadowCr4 == pCtx->cr4)
1707	{
1708	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1709	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1710	else
1711	{
1712	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1713	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1714	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1715	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1716	}
1717	}
1718	else
1719	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1720
1721	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking PGM mode changes. */
1722	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1723
1724	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1725	Assert(RT_HI_U32(uShadowCr4) == 0);
1726	pVmcb->guest.u64CR4 = uShadowCr4;
1727	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1728
1729	return VINF_SUCCESS;
1730	}
1731
1732
1733	/**
1734	* Exports the guest (or nested-guest) control registers into the VMCB.
1735	*
1736	* @returns VBox status code.
1737	* @param pVCpu The cross context virtual CPU structure.
1738	* @param pVmcb Pointer to the VM control block.
1739	*
1740	* @remarks No-long-jump zone!!!
1741	*/
1742	static int hmR0SvmExportGuestControlRegs(PVMCPU pVCpu, PSVMVMCB pVmcb)
1743	{
1744	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1745
1746	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1747	{
1748	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1749	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1750
1751	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1752	{
1753	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1754	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1755	}
1756
1757	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1758	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1759
1760	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1761	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1762	{
1763	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1764	if (RT_FAILURE(rc))
1765	return rc;
1766	}
1767
1768	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1769	}
1770	return VINF_SUCCESS;
1771	}
1772
1773
1774	/**
1775	* Exports the guest (or nested-guest) segment registers into the VMCB.
1776	*
1777	* @returns VBox status code.
1778	* @param pVCpu The cross context virtual CPU structure.
1779	* @param pVmcb Pointer to the VM control block.
1780	*
1781	* @remarks No-long-jump zone!!!
1782	*/
1783	static void hmR0SvmExportGuestSegmentRegs(PVMCPU pVCpu, PSVMVMCB pVmcb)
1784	{
1785	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1786	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1787
1788	/* Guest segment registers. */
1789	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1790	{
1791	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1792	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1793
1794	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1795	{
1796	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1797	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1798	}
1799
1800	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1801	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1802
1803	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1804	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1805
1806	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1807	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1808
1809	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1810	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1811
1812	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1813	}
1814
1815	/* Guest TR. */
1816	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1817	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1818
1819	/* Guest LDTR. */
1820	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1821	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1822
1823	/* Guest GDTR. */
1824	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1825	{
1826	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1827	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1828	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1829	}
1830
1831	/* Guest IDTR. */
1832	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1833	{
1834	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1835	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1836	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1837	}
1838
1839	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1840	\| HM_CHANGED_GUEST_TABLE_MASK);
1841	}
1842
1843
1844	/**
1845	* Exports the guest (or nested-guest) MSRs into the VMCB.
1846	*
1847	* @param pVCpu The cross context virtual CPU structure.
1848	* @param pVmcb Pointer to the VM control block.
1849	* @param pCtx Pointer to the guest-CPU context.
1850	*
1851	* @remarks No-long-jump zone!!!
1852	*/
1853	static void hmR0SvmExportGuestMsrs(PVMCPU pVCpu, PSVMVMCB pVmcb)
1854	{
1855	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1856	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1857
1858	/* Guest Sysenter MSRs. */
1859	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1860	{
1861	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1862	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1863
1864	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1865	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1866
1867	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1868	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1869	}
1870
1871	/*
1872	* Guest EFER MSR.
1873	* AMD-V requires guest EFER.SVME to be set. Weird.
1874	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1875	*/
1876	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1877	{
1878	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1879	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1880	}
1881
1882	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1883	if ( !CPUMIsGuestInLongModeEx(pCtx)
1884	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1885	{
1886	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1887	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1888	}
1889
1890	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1891	{
1892	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1893	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1894	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1895	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1896	}
1897
1898	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1899	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1900
1901	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1902	\| HM_CHANGED_GUEST_EFER_MSR
1903	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1904	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1905
1906	/*
1907	* Setup the PAT MSR (applicable for Nested Paging only).
1908	*
1909	* While guests can modify and see the modified values through the shadow values,
1910	* we shall not honor any guest modifications of this MSR to ensure caching is always
1911	* enabled similar to how we clear CR0.CD and NW bits.
1912	*
1913	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1914	*/
1915	pVmcb->guest.u64PAT = MSR_IA32_CR_PAT_INIT_VAL;
1916
1917	/* Enable the last branch record bit if LBR virtualization is enabled. */
1918	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1919	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1920	}
1921
1922
1923	/**
1924	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1925	* the necessary intercepts accordingly.
1926	*
1927	* @param pVCpu The cross context virtual CPU structure.
1928	* @param pVmcb Pointer to the VM control block.
1929	*
1930	* @remarks No-long-jump zone!!!
1931	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1932	*/
1933	static void hmR0SvmExportSharedDebugState(PVMCPU pVCpu, PSVMVMCB pVmcb)
1934	{
1935	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1936
1937	/*
1938	* Anyone single stepping on the host side? If so, we'll have to use the
1939	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1940	* the VMM level like the VT-x implementations does.
1941	*/
1942	bool fInterceptMovDRx = false;
1943	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1944	if (fStepping)
1945	{
1946	pVCpu->hm.s.fClearTrapFlag = true;
1947	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1948	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1949	}
1950
1951	if ( fStepping
1952	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1953	{
1954	/*
1955	* Use the combined guest and host DRx values found in the hypervisor
1956	* register set because the debugger has breakpoints active or someone
1957	* is single stepping on the host side.
1958	*
1959	* Note! DBGF expects a clean DR6 state before executing guest code.
1960	*/
1961	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1962	if ( CPUMIsGuestInLongModeEx(pCtx)
1963	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
1964	{
1965	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1966	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
1967	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
1968	}
1969	else
1970	#endif
1971	if (!CPUMIsHyperDebugStateActive(pVCpu))
1972	{
1973	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1974	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1975	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1976	}
1977
1978	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1979	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1980	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1981	{
1982	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1983	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1984	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1985	}
1986
1987	/** @todo If we cared, we could optimize to allow the guest to read registers
1988	* with the same values. */
1989	fInterceptMovDRx = true;
1990	pVCpu->hm.s.fUsingHyperDR7 = true;
1991	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1992	}
1993	else
1994	{
1995	/*
1996	* Update DR6, DR7 with the guest values if necessary.
1997	*/
1998	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1999	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
2000	{
2001	pVmcb->guest.u64DR7 = pCtx->dr[7];
2002	pVmcb->guest.u64DR6 = pCtx->dr[6];
2003	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
2004	}
2005	pVCpu->hm.s.fUsingHyperDR7 = false;
2006
2007	/*
2008	* If the guest has enabled debug registers, we need to load them prior to
2009	* executing guest code so they'll trigger at the right time.
2010	*/
2011	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
2012	{
2013	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
2014	if ( CPUMIsGuestInLongModeEx(pCtx)
2015	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
2016	{
2017	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
2018	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2019	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
2020	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
2021	}
2022	else
2023	#endif
2024	if (!CPUMIsGuestDebugStateActive(pVCpu))
2025	{
2026	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
2027	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2028	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2029	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2030	}
2031	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2032	}
2033	/*
2034	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2035	* intercept #DB as DR6 is updated in the VMCB.
2036	*
2037	* Note! If we cared and dared, we could skip intercepting \#DB here.
2038	* However, \#DB shouldn't be performance critical, so we'll play safe
2039	* and keep the code similar to the VT-x code and always intercept it.
2040	*/
2041	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
2042	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
2043	&& !CPUMIsGuestDebugStateActive(pVCpu))
2044	#else
2045	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2046	#endif
2047	{
2048	fInterceptMovDRx = true;
2049	}
2050	}
2051
2052	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2053	if (fInterceptMovDRx)
2054	{
2055	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2056	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2057	{
2058	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2059	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2060	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2061	}
2062	}
2063	else
2064	{
2065	if ( pVmcb->ctrl.u16InterceptRdDRx
2066	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2067	{
2068	pVmcb->ctrl.u16InterceptRdDRx = 0;
2069	pVmcb->ctrl.u16InterceptWrDRx = 0;
2070	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2071	}
2072	}
2073	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2074	}
2075
2076	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2077	/**
2078	* Exports the nested-guest hardware virtualization state into the nested-guest
2079	* VMCB.
2080	*
2081	* @param pVCpu The cross context virtual CPU structure.
2082	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2083	*
2084	* @remarks No-long-jump zone!!!
2085	*/
2086	static void hmR0SvmExportGuestHwvirtStateNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst)
2087	{
2088	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2089
2090	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2091	{
2092	/*
2093	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2094	* since SVM doesn't have a preemption timer.
2095	*
2096	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2097	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2098	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2099	*/
2100	PVM pVM = pVCpu->CTX_SUFF(pVM);
2101	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2102	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2103	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2104	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_PAUSE))
2105	{
2106	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2107	pVmcbNstGstCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2108	pVmcbNstGstCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2109	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2110	}
2111	else
2112	{
2113	pVmcbNstGstCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2114	pVmcbNstGstCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2115	}
2116
2117	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2118	}
2119	}
2120	#endif
2121
2122	/**
2123	* Exports the guest APIC TPR state into the VMCB.
2124	*
2125	* @returns VBox status code.
2126	* @param pVCpu The cross context virtual CPU structure.
2127	* @param pVmcb Pointer to the VM control block.
2128	*/
2129	static int hmR0SvmExportGuestApicTpr(PVMCPU pVCpu, PSVMVMCB pVmcb)
2130	{
2131	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2132	{
2133	PVM pVM = pVCpu->CTX_SUFF(pVM);
2134	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2135	if ( PDMHasApic(pVM)
2136	&& APICIsEnabled(pVCpu))
2137	{
2138	bool fPendingIntr;
2139	uint8_t u8Tpr;
2140	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2141	AssertRCReturn(rc, rc);
2142
2143	/* Assume that we need to trap all TPR accesses and thus need not check on
2144	every #VMEXIT if we should update the TPR. */
2145	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2146	pVCpu->hm.s.svm.fSyncVTpr = false;
2147
2148	if (!pVM->hm.s.fTPRPatchingActive)
2149	{
2150	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2151	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2152
2153	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2154	can deliver the interrupt to the guest. */
2155	if (fPendingIntr)
2156	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2157	else
2158	{
2159	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2160	pVCpu->hm.s.svm.fSyncVTpr = true;
2161	}
2162
2163	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2164	}
2165	else
2166	{
2167	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2168	pVmcb->guest.u64LSTAR = u8Tpr;
2169	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
2170
2171	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2172	if (fPendingIntr)
2173	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2174	else
2175	{
2176	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2177	pVCpu->hm.s.svm.fSyncVTpr = true;
2178	}
2179	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2180	}
2181	}
2182	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2183	}
2184	return VINF_SUCCESS;
2185	}
2186
2187
2188	/**
2189	* Sets up the exception interrupts required for guest (or nested-guest)
2190	* execution in the VMCB.
2191	*
2192	* @param pVCpu The cross context virtual CPU structure.
2193	* @param pVmcb Pointer to the VM control block.
2194	*
2195	* @remarks No-long-jump zone!!!
2196	*/
2197	static void hmR0SvmExportGuestXcptIntercepts(PVMCPU pVCpu, PSVMVMCB pVmcb)
2198	{
2199	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2200
2201	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2202	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_GUEST_XCPT_INTERCEPTS)
2203	{
2204	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2205	if (pVCpu->hm.s.fGIMTrapXcptUD)
2206	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2207	else
2208	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2209
2210	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2211	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2212	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2213	else
2214	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2215
2216	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2217	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_SVM_GUEST_XCPT_INTERCEPTS;
2218	}
2219	}
2220
2221
2222	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2223	/**
2224	* Merges guest and nested-guest intercepts for executing the nested-guest using
2225	* hardware-assisted SVM.
2226	*
2227	* This merges the guest and nested-guest intercepts in a way that if the outer
2228	* guest intercept is set we need to intercept it in the nested-guest as
2229	* well.
2230	*
2231	* @param pVCpu The cross context virtual CPU structure.
2232	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2233	* @param pCtx Pointer to the nested-guest-CPU context.
2234	*/
2235	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPU pVCpu, PCCPUMCTX pCtx)
2236	{
2237	PVM pVM = pVCpu->CTX_SUFF(pVM);
2238	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2239	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2240	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2241
2242	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2243	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2244	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2245
2246	/* Always intercept CR4 writes for tracking PGM mode changes. */
2247	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2248
2249	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2250	if (!pVM->hm.s.fNestedPaging)
2251	{
2252	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2253	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2254	}
2255
2256	/** @todo Figure out debugging with nested-guests, till then just intercept
2257	* all DR[0-15] accesses. */
2258	pVmcbNstGstCtrl->u16InterceptRdDRx \|= 0xffff;
2259	pVmcbNstGstCtrl->u16InterceptWrDRx \|= 0xffff;
2260
2261	/*
2262	* Merge the guest's exception intercepts into the nested-guest VMCB.
2263	*
2264	* - \#UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2265	* while executing the nested-guest.
2266	*
2267	* - \#BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2268	* be tweaked later depending on how we wish to implement breakpoints.
2269	*
2270	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2271	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2272	*/
2273	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2274	pVmcbNstGstCtrl->u32InterceptXcpt \|= (pVmcb->ctrl.u32InterceptXcpt & ~( RT_BIT(X86_XCPT_UD)
2275	\| RT_BIT(X86_XCPT_BP)));
2276	#else
2277	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2278	#endif
2279
2280	/*
2281	* Adjust intercepts while executing the nested-guest that differ from the
2282	* outer guest intercepts.
2283	*
2284	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2285	* that belong to the nested-guest to the outer guest.
2286	*
2287	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2288	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2289	*/
2290	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2291	\| SVM_CTRL_INTERCEPT_VMMCALL))
2292	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2293
2294	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2295	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2296
2297	/* Finally, update the VMCB clean bits. */
2298	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2299	}
2300	#endif
2301
2302
2303	/**
2304	* Selects the appropriate function to run guest code.
2305	*
2306	* @returns VBox status code.
2307	* @param pVCpu The cross context virtual CPU structure.
2308	*
2309	* @remarks No-long-jump zone!!!
2310	*/
2311	static int hmR0SvmSelectVMRunHandler(PVMCPU pVCpu)
2312	{
2313	if (CPUMIsGuestInLongMode(pVCpu))
2314	{
2315	#ifndef VBOX_ENABLE_64_BITS_GUESTS
2316	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2317	#endif
2318	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
2319	#if HC_ARCH_BITS == 32
2320	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
2321	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
2322	#else
2323	/* 64-bit host or hybrid host. */
2324	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun64;
2325	#endif
2326	}
2327	else
2328	{
2329	/* Guest is not in long mode, use the 32-bit handler. */
2330	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
2331	}
2332	return VINF_SUCCESS;
2333	}
2334
2335
2336	/**
2337	* Enters the AMD-V session.
2338	*
2339	* @returns VBox status code.
2340	* @param pVM The cross context VM structure.
2341	* @param pVCpu The cross context virtual CPU structure.
2342	* @param pCpu Pointer to the CPU info struct.
2343	*/
2344	VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2345	{
2346	AssertPtr(pVM);
2347	AssertPtr(pVCpu);
2348	Assert(pVM->hm.s.svm.fSupported);
2349	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2350	NOREF(pVM); NOREF(pCpu);
2351
2352	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2353	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2354	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2355
2356	pVCpu->hm.s.fLeaveDone = false;
2357	return VINF_SUCCESS;
2358	}
2359
2360
2361	/**
2362	* Thread-context callback for AMD-V.
2363	*
2364	* @param enmEvent The thread-context event.
2365	* @param pVCpu The cross context virtual CPU structure.
2366	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2367	* @thread EMT(pVCpu)
2368	*/
2369	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
2370	{
2371	NOREF(fGlobalInit);
2372
2373	switch (enmEvent)
2374	{
2375	case RTTHREADCTXEVENT_OUT:
2376	{
2377	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2378	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2379	VMCPU_ASSERT_EMT(pVCpu);
2380
2381	/* No longjmps (log-flush, locks) in this fragile context. */
2382	VMMRZCallRing3Disable(pVCpu);
2383
2384	if (!pVCpu->hm.s.fLeaveDone)
2385	{
2386	hmR0SvmLeave(pVCpu, false /* fImportState */);
2387	pVCpu->hm.s.fLeaveDone = true;
2388	}
2389
2390	/* Leave HM context, takes care of local init (term). */
2391	int rc = HMR0LeaveCpu(pVCpu);
2392	AssertRC(rc); NOREF(rc);
2393
2394	/* Restore longjmp state. */
2395	VMMRZCallRing3Enable(pVCpu);
2396	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2397	break;
2398	}
2399
2400	case RTTHREADCTXEVENT_IN:
2401	{
2402	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2403	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2404	VMCPU_ASSERT_EMT(pVCpu);
2405
2406	/* No longjmps (log-flush, locks) in this fragile context. */
2407	VMMRZCallRing3Disable(pVCpu);
2408
2409	/*
2410	* Initialize the bare minimum state required for HM. This takes care of
2411	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2412	*/
2413	int rc = HMR0EnterCpu(pVCpu);
2414	AssertRC(rc); NOREF(rc);
2415	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2416	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2417
2418	pVCpu->hm.s.fLeaveDone = false;
2419
2420	/* Restore longjmp state. */
2421	VMMRZCallRing3Enable(pVCpu);
2422	break;
2423	}
2424
2425	default:
2426	break;
2427	}
2428	}
2429
2430
2431	/**
2432	* Saves the host state.
2433	*
2434	* @returns VBox status code.
2435	* @param pVM The cross context VM structure.
2436	* @param pVCpu The cross context virtual CPU structure.
2437	*
2438	* @remarks No-long-jump zone!!!
2439	*/
2440	VMMR0DECL(int) SVMR0ExportHostState(PVMCPU pVCpu)
2441	{
2442	NOREF(pVCpu);
2443
2444	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2445	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2446	return VINF_SUCCESS;
2447	}
2448
2449
2450	/**
2451	* Exports the guest state from the guest-CPU context into the VMCB.
2452	*
2453	* The CPU state will be loaded from these fields on every successful VM-entry.
2454	* Also sets up the appropriate VMRUN function to execute guest code based on
2455	* the guest CPU mode.
2456	*
2457	* @returns VBox status code.
2458	* @param pVM The cross context VM structure.
2459	* @param pVCpu The cross context virtual CPU structure.
2460	* @param pCtx Pointer to the guest-CPU context.
2461	*
2462	* @remarks No-long-jump zone!!!
2463	*/
2464	static int hmR0SvmExportGuestState(PVMCPU pVCpu)
2465	{
2466	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2467
2468	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2469	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2470
2471	Assert(pVmcb);
2472	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
2473
2474	pVmcb->guest.u64RIP = pCtx->rip;
2475	pVmcb->guest.u64RSP = pCtx->rsp;
2476	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2477	pVmcb->guest.u64RAX = pCtx->rax;
2478	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2479	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2480	{
2481	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);
2482	pVmcb->ctrl.IntCtrl.n.u1VGif = pCtx->hwvirt.fGif;
2483	}
2484	#endif
2485
2486	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2487
2488	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2489	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2490
2491	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2492	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2493	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2494
2495	ASMSetFlags(fEFlags);
2496
2497	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2498	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2499	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2500
2501	rc = hmR0SvmSelectVMRunHandler(pVCpu);
2502	AssertRCReturn(rc, rc);
2503
2504	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2505	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( HM_CHANGED_GUEST_RIP
2506	\| HM_CHANGED_GUEST_RFLAGS
2507	\| HM_CHANGED_GUEST_GPRS_MASK
2508	\| HM_CHANGED_GUEST_X87
2509	\| HM_CHANGED_GUEST_SSE_AVX
2510	\| HM_CHANGED_GUEST_OTHER_XSAVE
2511	\| HM_CHANGED_GUEST_XCRx
2512	\| HM_CHANGED_GUEST_TSC_AUX
2513	\| HM_CHANGED_GUEST_OTHER_MSRS
2514	\| HM_CHANGED_GUEST_HWVIRT
2515	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_GUEST_XCPT_INTERCEPTS)));
2516
2517	#ifdef VBOX_STRICT
2518	/*
2519	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2520	* except for the host-context and/or shared host-guest context bits.
2521	*/
2522	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2523	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2524	("fCtxChanged=%#RX64\n", fCtxChanged));
2525
2526	/*
2527	* If we need to log state that isn't always imported, we'll need to import them here.
2528	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2529	*/
2530	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2531	#endif
2532
2533	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2534	return VINF_SUCCESS;
2535	}
2536
2537
2538	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2539	/**
2540	* Merges the guest and nested-guest MSR permission bitmap.
2541	*
2542	* If the guest is intercepting an MSR we need to intercept it regardless of
2543	* whether the nested-guest is intercepting it or not.
2544	*
2545	* @param pHostCpu Pointer to the physical CPU HM info. struct.
2546	* @param pVCpu The cross context virtual CPU structure.
2547	* @param pCtx Pointer to the nested-guest-CPU context.
2548	*
2549	* @remarks No-long-jmp zone!!!
2550	*/
2551	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMGLOBALCPUINFO pHostCpu, PVMCPU pVCpu, PCCPUMCTX pCtx)
2552	{
2553	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hm.s.svm.pvMsrBitmap;
2554	uint64_t const pu64NstGstMsrpm = (uint64_t const )pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
2555	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2556
2557	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2558	uint32_t const offRsvdQwords = 0x1800 >> 3;
2559	for (uint32_t i = 0; i < offRsvdQwords; i++)
2560	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2561	}
2562
2563
2564	/**
2565	* Caches the nested-guest VMCB fields before we modify them for execution using
2566	* hardware-assisted SVM.
2567	*
2568	* @returns true if the VMCB was previously already cached, false otherwise.
2569	* @param pCtx Pointer to the guest-CPU context.
2570	*
2571	* @sa HMSvmNstGstVmExitNotify.
2572	*/
2573	static bool hmR0SvmCacheVmcbNested(PVMCPU pVCpu, PCCPUMCTX pCtx)
2574	{
2575	/*
2576	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2577	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2578	*
2579	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2580	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2581	*/
2582	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2583	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2584	if (!fWasCached)
2585	{
2586	PCSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2587	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2588	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2589	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2590	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2591	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2592	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2593	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2594	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2595	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2596	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2597	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2598	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2599	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2600	pVmcbNstGstCache->fCacheValid = true;
2601	Log4Func(("Cached VMCB fields\n"));
2602	}
2603
2604	return fWasCached;
2605	}
2606
2607
2608	/**
2609	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2610	*
2611	* This is done the first time we enter nested-guest execution using SVM R0
2612	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2613	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2614	*
2615	* @param pVCpu The cross context virtual CPU structure.
2616	*/
2617	static void hmR0SvmSetupVmcbNested(PVMCPU pVCpu)
2618	{
2619	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2620	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2621	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2622
2623	/*
2624	* First cache the nested-guest VMCB fields we may potentially modify.
2625	*/
2626	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu, pCtx);
2627	if (!fVmcbCached)
2628	{
2629	/*
2630	* The IOPM of the nested-guest can be ignored because the the guest always
2631	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2632	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2633	*/
2634	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2635
2636	/*
2637	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2638	* nested-paging suddenly while executing a VM (see assertion at the end of
2639	* Trap0eHandler() in PGMAllBth.h).
2640	*/
2641	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
2642
2643	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2644	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2645
2646	/*
2647	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2648	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2649	*/
2650	pVCpu->hm.s.svm.fSyncVTpr = false;
2651
2652	#ifdef DEBUG_ramshankar
2653	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2654	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2655	#endif
2656
2657	/*
2658	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2659	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2660	*/
2661	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2662	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2663	\| SVM_CTRL_INTERCEPT_VMLOAD;
2664
2665	/*
2666	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2667	* CLGI/STGI instructions executed by the nested-guest.
2668	*/
2669	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2670	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2671	\| SVM_CTRL_INTERCEPT_STGI;
2672
2673	/* Merge the guest and nested-guest intercepts. */
2674	hmR0SvmMergeVmcbCtrlsNested(pVCpu, pCtx);
2675
2676	/* Update the VMCB clean bits. */
2677	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2678	}
2679	else
2680	{
2681	Assert(!pVCpu->hm.s.svm.fSyncVTpr);
2682	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2683	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
2684	}
2685	}
2686
2687
2688	/**
2689	* Exports the nested-guest state into the VMCB.
2690	*
2691	* We need to export the entire state as we could be continuing nested-guest
2692	* execution at any point (not just immediately after VMRUN) and thus the VMCB
2693	* can be out-of-sync with the nested-guest state if it was executed in IEM.
2694	*
2695	* @returns VBox status code.
2696	* @param pVCpu The cross context virtual CPU structure.
2697	* @param pCtx Pointer to the guest-CPU context.
2698	*
2699	* @remarks No-long-jump zone!!!
2700	*/
2701	static int hmR0SvmExportGuestStateNested(PVMCPU pVCpu)
2702	{
2703	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2704
2705	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2706	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2707	Assert(pVmcbNstGst);
2708
2709	hmR0SvmSetupVmcbNested(pVCpu);
2710
2711	pVmcbNstGst->guest.u64RIP = pCtx->rip;
2712	pVmcbNstGst->guest.u64RSP = pCtx->rsp;
2713	pVmcbNstGst->guest.u64RFlags = pCtx->eflags.u32;
2714	pVmcbNstGst->guest.u64RAX = pCtx->rax;
2715
2716	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2717
2718	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcbNstGst);
2719	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2720
2721	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcbNstGst);
2722	hmR0SvmExportGuestMsrs(pVCpu, pVmcbNstGst);
2723	hmR0SvmExportGuestHwvirtStateNested(pVCpu, pVmcbNstGst);
2724
2725	ASMSetFlags(fEFlags);
2726
2727	/* Nested VGIF not supported yet. */
2728	Assert(!pVmcbNstGst->ctrl.IntCtrl.n.u1VGifEnable);
2729
2730	rc = hmR0SvmSelectVMRunHandler(pVCpu);
2731	AssertRCReturn(rc, rc);
2732
2733	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2734	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( HM_CHANGED_GUEST_RIP
2735	\| HM_CHANGED_GUEST_RFLAGS
2736	\| HM_CHANGED_GUEST_GPRS_MASK
2737	\| HM_CHANGED_GUEST_APIC_TPR
2738	\| HM_CHANGED_GUEST_X87
2739	\| HM_CHANGED_GUEST_SSE_AVX
2740	\| HM_CHANGED_GUEST_OTHER_XSAVE
2741	\| HM_CHANGED_GUEST_XCRx
2742	\| HM_CHANGED_GUEST_TSC_AUX
2743	\| HM_CHANGED_GUEST_OTHER_MSRS
2744	\| HM_CHANGED_SVM_GUEST_XCPT_INTERCEPTS
2745	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2746
2747	#ifdef VBOX_STRICT
2748	/*
2749	* All of the guest-CPU state and SVM keeper bits should be exported here by now, except
2750	* for the host-context and/or shared host-guest context bits.
2751	*/
2752	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2753	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2754	("fCtxChanged=%#RX64\n", fCtxChanged));
2755
2756	/*
2757	* If we need to log state that isn't always imported, we'll need to import them here.
2758	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2759	*/
2760	hmR0SvmLogState(pVCpu, pVmcbNstGst, "hmR0SvmExportGuestStateNested", 0 /* fFlags /, 0 / uVerbose */);
2761	#endif
2762
2763	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2764	return rc;
2765	}
2766	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2767
2768
2769	/**
2770	* Exports the state shared between the host and guest (or nested-guest) into
2771	* the VMCB.
2772	*
2773	* @param pVCpu The cross context virtual CPU structure.
2774	* @param pVmcb Pointer to the VM control block.
2775	*
2776	* @remarks No-long-jump zone!!!
2777	*/
2778	static void hmR0SvmExportSharedState(PVMCPU pVCpu, PSVMVMCB pVmcb)
2779	{
2780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2781	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2782
2783	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2784	{
2785	/** @todo Figure out stepping with nested-guest. */
2786	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2787	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
2788	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2789	else
2790	{
2791	pVmcb->guest.u64DR6 = pCtx->dr[6];
2792	pVmcb->guest.u64DR7 = pCtx->dr[7];
2793	}
2794	}
2795
2796	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2797	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2798	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2799	}
2800
2801
2802	/**
2803	* Worker for SVMR0ImportStateOnDemand.
2804	*
2805	* @param pVCpu The cross context virtual CPU structure.
2806	* @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
2807	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2808	*/
2809	static void hmR0SvmImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
2810	{
2811	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2812
2813	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2814	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
2815	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2816	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2817
2818	Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
2819
2820	/*
2821	* We disable interrupts to make the updating of the state and in particular
2822	* the fExtrn modification atomic wrt to preemption hooks.
2823	*/
2824	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2825
2826	fWhat &= pCtx->fExtrn;
2827	if (fWhat)
2828	{
2829	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2830	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2831	{
2832	if ( !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
2833	&& pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2834	{
2835	/* We don't yet support passing VGIF feature to the guest. */
2836	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fVGif);
2837	pCtx->hwvirt.fGif = pVmcbCtrl->IntCtrl.n.u1VGif;
2838	}
2839	}
2840
2841	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2842	{
2843	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2844	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2845	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2846	}
2847	#endif
2848
2849	if (fWhat & CPUMCTX_EXTRN_HM_SVM_INT_SHADOW)
2850	{
2851	if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2852	EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
2853	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2854	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2855	}
2856
2857	if (fWhat & CPUMCTX_EXTRN_RIP)
2858	pCtx->rip = pVmcbGuest->u64RIP;
2859
2860	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2861	pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
2862
2863	if (fWhat & CPUMCTX_EXTRN_RSP)
2864	pCtx->rsp = pVmcbGuest->u64RSP;
2865
2866	if (fWhat & CPUMCTX_EXTRN_RAX)
2867	pCtx->rax = pVmcbGuest->u64RAX;
2868
2869	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2870	{
2871	if (fWhat & CPUMCTX_EXTRN_CS)
2872	{
2873	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2874	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2875	/** @todo SELM might need to be fixed as it too should not care about the
2876	* granularity bit. See @bugref{6785}. */
2877	if ( !pCtx->cs.Attr.n.u1Granularity
2878	&& pCtx->cs.Attr.n.u1Present
2879	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2880	{
2881	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2882	pCtx->cs.Attr.n.u1Granularity = 1;
2883	}
2884	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2885	}
2886	if (fWhat & CPUMCTX_EXTRN_SS)
2887	{
2888	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2889	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2890	/*
2891	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2892	* VMCB and uses that and thus it's possible that when the CPL changes during
2893	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2894	* AMD Fusion CPUs with 64-bit guests.
2895	*
2896	* See AMD spec. 15.5.1 "Basic operation".
2897	*/
2898	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2899	uint8_t const uCpl = pVmcbGuest->u8CPL;
2900	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2901	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2902	}
2903	if (fWhat & CPUMCTX_EXTRN_DS)
2904	{
2905	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2906	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2907	}
2908	if (fWhat & CPUMCTX_EXTRN_ES)
2909	{
2910	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2911	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2912	}
2913	if (fWhat & CPUMCTX_EXTRN_FS)
2914	{
2915	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2916	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2917	}
2918	if (fWhat & CPUMCTX_EXTRN_GS)
2919	{
2920	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2921	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2922	}
2923	}
2924
2925	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2926	{
2927	if (fWhat & CPUMCTX_EXTRN_TR)
2928	{
2929	/*
2930	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2931	* are used between Intel and AMD, see @bugref{6208#c39}.
2932	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2933	*/
2934	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2935	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2936	{
2937	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2938	\|\| CPUMIsGuestInLongModeEx(pCtx))
2939	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2940	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2941	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2942	}
2943	}
2944
2945	if (fWhat & CPUMCTX_EXTRN_LDTR)
2946	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2947
2948	if (fWhat & CPUMCTX_EXTRN_GDTR)
2949	{
2950	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2951	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2952	}
2953
2954	if (fWhat & CPUMCTX_EXTRN_IDTR)
2955	{
2956	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2957	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2958	}
2959	}
2960
2961	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2962	{
2963	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2964	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2965	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2966	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2967	}
2968
2969	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2970	{
2971	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2972	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2973	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2974	}
2975
2976	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2977	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2978
2979	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2980	{
2981	if (fWhat & CPUMCTX_EXTRN_DR6)
2982	{
2983	if (!pVCpu->hm.s.fUsingHyperDR7)
2984	pCtx->dr[6] = pVmcbGuest->u64DR6;
2985	else
2986	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2987	}
2988
2989	if (fWhat & CPUMCTX_EXTRN_DR7)
2990	{
2991	if (!pVCpu->hm.s.fUsingHyperDR7)
2992	pCtx->dr[7] = pVmcbGuest->u64DR7;
2993	else
2994	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2995	}
2996	}
2997
2998	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2999	{
3000	if (fWhat & CPUMCTX_EXTRN_CR0)
3001	{
3002	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
3003	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
3004	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
3005	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
3006	CPUMSetGuestCR0(pVCpu, uCr0);
3007	VMMRZCallRing3Enable(pVCpu);
3008	}
3009
3010	if (fWhat & CPUMCTX_EXTRN_CR2)
3011	pCtx->cr2 = pVmcbGuest->u64CR2;
3012
3013	if (fWhat & CPUMCTX_EXTRN_CR3)
3014	{
3015	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
3016	&& pCtx->cr3 != pVmcbGuest->u64CR3)
3017	{
3018	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
3019	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3020	}
3021	}
3022
3023	/* Changes to CR4 are always intercepted. */
3024	}
3025
3026	/* Update fExtrn. */
3027	pCtx->fExtrn &= ~fWhat;
3028
3029	/* If everything has been imported, clear the HM keeper bit. */
3030	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
3031	{
3032	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
3033	Assert(!pCtx->fExtrn);
3034	}
3035	}
3036	else
3037	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3038
3039	ASMSetFlags(fEFlags);
3040
3041	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
3042
3043	/*
3044	* Honor any pending CR3 updates.
3045	*
3046	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
3047	* -> hmR0SvmCallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
3048	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
3049	* handling -> hmR0SvmImportGuestState() and here we are.
3050	*
3051	* The reason for such complicated handling is because VM-exits that call into PGM expect
3052	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
3053	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
3054	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
3055	*
3056	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
3057	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
3058	*/
3059	if ( VMMRZCallRing3IsEnabled(pVCpu)
3060	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3061	{
3062	Assert(pCtx->cr3 == pVmcbGuest->u64CR3);
3063	PGMUpdateCR3(pVCpu, pCtx->cr3);
3064	}
3065	}
3066
3067
3068	/**
3069	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
3070	* context.
3071	*
3072	* Currently there is no residual state left in the CPU that is not updated in the
3073	* VMCB.
3074	*
3075	* @returns VBox status code.
3076	* @param pVCpu The cross context virtual CPU structure.
3077	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3078	*/
3079	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
3080	{
3081	hmR0SvmImportGuestState(pVCpu, fWhat);
3082	return VINF_SUCCESS;
3083	}
3084
3085
3086	/**
3087	* Does the necessary state syncing before returning to ring-3 for any reason
3088	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
3089	*
3090	* @param pVCpu The cross context virtual CPU structure.
3091	* @param fImportState Whether to import the guest state from the VMCB back
3092	* to the guest-CPU context.
3093	*
3094	* @remarks No-long-jmp zone!!!
3095	*/
3096	static void hmR0SvmLeave(PVMCPU pVCpu, bool fImportState)
3097	{
3098	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3099	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3100	Assert(VMMR0IsLogFlushDisabled(pVCpu));
3101
3102	/*
3103	* !!! IMPORTANT !!!
3104	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
3105	*/
3106
3107	/* Save the guest state if necessary. */
3108	if (fImportState)
3109	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3110
3111	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3112	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3113	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
3114
3115	/*
3116	* Restore host debug registers if necessary and resync on next R0 reentry.
3117	*/
3118	#ifdef VBOX_STRICT
3119	if (CPUMIsHyperDebugStateActive(pVCpu))
3120	{
3121	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb; /** @todo nested-guest. */
3122	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
3123	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
3124	}
3125	#endif
3126	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3127	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3128	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3129
3130	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3131	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3132	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3133	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
3134	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
3135	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3136
3137	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3138	}
3139
3140
3141	/**
3142	* Leaves the AMD-V session.
3143	*
3144	* Only used while returning to ring-3 either due to longjump or exits to
3145	* ring-3.
3146	*
3147	* @returns VBox status code.
3148	* @param pVCpu The cross context virtual CPU structure.
3149	*/
3150	static int hmR0SvmLeaveSession(PVMCPU pVCpu)
3151	{
3152	HM_DISABLE_PREEMPT();
3153	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3154	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3155
3156	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3157	and done this from the SVMR0ThreadCtxCallback(). */
3158	if (!pVCpu->hm.s.fLeaveDone)
3159	{
3160	hmR0SvmLeave(pVCpu, true /* fImportState */);
3161	pVCpu->hm.s.fLeaveDone = true;
3162	}
3163
3164	/*
3165	* !!! IMPORTANT !!!
3166	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
3167	*/
3168
3169	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3170	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3171	VMMR0ThreadCtxHookDisable(pVCpu);
3172
3173	/* Leave HM context. This takes care of local init (term). */
3174	int rc = HMR0LeaveCpu(pVCpu);
3175
3176	HM_RESTORE_PREEMPT();
3177	return rc;
3178	}
3179
3180
3181	/**
3182	* Does the necessary state syncing before doing a longjmp to ring-3.
3183	*
3184	* @returns VBox status code.
3185	* @param pVCpu The cross context virtual CPU structure.
3186	*
3187	* @remarks No-long-jmp zone!!!
3188	*/
3189	static int hmR0SvmLongJmpToRing3(PVMCPU pVCpu)
3190	{
3191	return hmR0SvmLeaveSession(pVCpu);
3192	}
3193
3194
3195	/**
3196	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3197	* any remaining host state) before we longjump to ring-3 and possibly get
3198	* preempted.
3199	*
3200	* @param pVCpu The cross context virtual CPU structure.
3201	* @param enmOperation The operation causing the ring-3 longjump.
3202	* @param pvUser The user argument (pointer to the possibly
3203	* out-of-date guest-CPU context).
3204	*/
3205	static DECLCALLBACK(int) hmR0SvmCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
3206	{
3207	RT_NOREF_PV(pvUser);
3208
3209	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
3210	{
3211	/*
3212	* !!! IMPORTANT !!!
3213	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3214	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3215	*/
3216	VMMRZCallRing3RemoveNotification(pVCpu);
3217	VMMRZCallRing3Disable(pVCpu);
3218	HM_DISABLE_PREEMPT();
3219
3220	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3221	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3222
3223	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3224	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3225
3226	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3227	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3228	VMMR0ThreadCtxHookDisable(pVCpu);
3229
3230	/* Leave HM context. This takes care of local init (term). */
3231	HMR0LeaveCpu(pVCpu);
3232
3233	HM_RESTORE_PREEMPT();
3234	return VINF_SUCCESS;
3235	}
3236
3237	Assert(pVCpu);
3238	Assert(pvUser);
3239	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3240	HMSVM_ASSERT_PREEMPT_SAFE();
3241
3242	VMMRZCallRing3Disable(pVCpu);
3243	Assert(VMMR0IsLogFlushDisabled(pVCpu));
3244
3245	Log4Func(("Calling hmR0SvmLongJmpToRing3\n"));
3246	int rc = hmR0SvmLongJmpToRing3(pVCpu);
3247	AssertRCReturn(rc, rc);
3248
3249	VMMRZCallRing3Enable(pVCpu);
3250	return VINF_SUCCESS;
3251	}
3252
3253
3254	/**
3255	* Take necessary actions before going back to ring-3.
3256	*
3257	* An action requires us to go back to ring-3. This function does the necessary
3258	* steps before we can safely return to ring-3. This is not the same as longjmps
3259	* to ring-3, this is voluntary.
3260	*
3261	* @returns VBox status code.
3262	* @param pVM The cross context VM structure.
3263	* @param pVCpu The cross context virtual CPU structure.
3264	* @param pCtx Pointer to the guest-CPU context.
3265	* @param rcExit The reason for exiting to ring-3. Can be
3266	* VINF_VMM_UNKNOWN_RING3_CALL.
3267	*/
3268	static int hmR0SvmExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rcExit)
3269	{
3270	Assert(pVM);
3271	Assert(pVCpu);
3272	Assert(pCtx);
3273	HMSVM_ASSERT_PREEMPT_SAFE();
3274
3275	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3276	VMMRZCallRing3Disable(pVCpu);
3277	Log4Func(("rcExit=%d LocalFF=%#RX32 GlobalFF=%#RX32\n", rcExit, pVCpu->fLocalForcedActions, pVM->fGlobalForcedActions));
3278
3279	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3280	if (pVCpu->hm.s.Event.fPending)
3281	{
3282	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3283	Assert(!pVCpu->hm.s.Event.fPending);
3284	}
3285
3286	/* Sync. the necessary state for going back to ring-3. */
3287	hmR0SvmLeaveSession(pVCpu);
3288	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3289
3290	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3291	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3292	\| CPUM_CHANGED_LDTR
3293	\| CPUM_CHANGED_GDTR
3294	\| CPUM_CHANGED_IDTR
3295	\| CPUM_CHANGED_TR
3296	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3297	if ( pVM->hm.s.fNestedPaging
3298	&& CPUMIsGuestPagingEnabledEx(pCtx))
3299	{
3300	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3301	}
3302
3303	/* Update the exit-to-ring 3 reason. */
3304	pVCpu->hm.s.rcLastExitToR3 = rcExit;
3305
3306	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
3307	if (rcExit != VINF_EM_RAW_INTERRUPT)
3308	{
3309	Assert(!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3310	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3311	}
3312
3313	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3314
3315	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
3316	VMMRZCallRing3RemoveNotification(pVCpu);
3317	VMMRZCallRing3Enable(pVCpu);
3318
3319	/*
3320	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3321	* and if we're injecting an event we should have a TRPM trap pending.
3322	*/
3323	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3324	pVCpu->hm.s.u32HMError = rcExit,
3325	VERR_SVM_IPE_5);
3326	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3327	pVCpu->hm.s.u32HMError = rcExit,
3328	VERR_SVM_IPE_4);
3329
3330	return rcExit;
3331	}
3332
3333
3334	/**
3335	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3336	* intercepts.
3337	*
3338	* @param pVM The cross context VM structure.
3339	* @param pVCpu The cross context virtual CPU structure.
3340	* @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
3341	* @param pVmcb Pointer to the VM control block.
3342	*
3343	* @remarks No-long-jump zone!!!
3344	*/
3345	static void hmR0SvmUpdateTscOffsetting(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pCtx, PSVMVMCB pVmcb)
3346	{
3347	/*
3348	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3349	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3350	* RDTSC/RDTSCP as well.
3351	*/
3352	bool fParavirtTsc;
3353	uint64_t uTscOffset;
3354	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
3355
3356	bool fIntercept;
3357	if (fCanUseRealTsc)
3358	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pCtx, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3359	else
3360	{
3361	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3362	fIntercept = true;
3363	}
3364
3365	if (!fIntercept)
3366	{
3367	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3368	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3369	uTscOffset = HMSvmNstGstApplyTscOffset(pVCpu, uTscOffset);
3370
3371	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3372	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3373	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3374
3375	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
3376	}
3377	else
3378	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
3379
3380	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3381	information before every VM-entry, hence we have nothing to do here at the moment. */
3382	if (fParavirtTsc)
3383	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3384	}
3385
3386
3387	/**
3388	* Sets an event as a pending event to be injected into the guest.
3389	*
3390	* @param pVCpu The cross context virtual CPU structure.
3391	* @param pEvent Pointer to the SVM event.
3392	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3393	* page-fault.
3394	*
3395	* @remarks Statistics counter assumes this is a guest event being reflected to
3396	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3397	*/
3398	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPU pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3399	{
3400	Assert(!pVCpu->hm.s.Event.fPending);
3401	Assert(pEvent->n.u1Valid);
3402
3403	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3404	pVCpu->hm.s.Event.fPending = true;
3405	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3406
3407	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3408	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3409	}
3410
3411
3412	/**
3413	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3414	*
3415	* @param pVCpu The cross context virtual CPU structure.
3416	*/
3417	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPU pVCpu)
3418	{
3419	SVMEVENT Event;
3420	Event.u = 0;
3421	Event.n.u1Valid = 1;
3422	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3423	Event.n.u8Vector = X86_XCPT_UD;
3424	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3425	}
3426
3427
3428	/**
3429	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3430	*
3431	* @param pVCpu The cross context virtual CPU structure.
3432	*/
3433	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPU pVCpu)
3434	{
3435	SVMEVENT Event;
3436	Event.u = 0;
3437	Event.n.u1Valid = 1;
3438	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3439	Event.n.u8Vector = X86_XCPT_DB;
3440	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3441	}
3442
3443
3444	/**
3445	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3446	*
3447	* @param pVCpu The cross context virtual CPU structure.
3448	* @param pCtx Pointer to the guest-CPU context.
3449	* @param u32ErrCode The error-code for the page-fault.
3450	* @param uFaultAddress The page fault address (CR2).
3451	*
3452	* @remarks This updates the guest CR2 with @a uFaultAddress!
3453	*/
3454	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3455	{
3456	SVMEVENT Event;
3457	Event.u = 0;
3458	Event.n.u1Valid = 1;
3459	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3460	Event.n.u8Vector = X86_XCPT_PF;
3461	Event.n.u1ErrorCodeValid = 1;
3462	Event.n.u32ErrorCode = u32ErrCode;
3463
3464	/* Update CR2 of the guest. */
3465	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3466	if (pCtx->cr2 != uFaultAddress)
3467	{
3468	pCtx->cr2 = uFaultAddress;
3469	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3470	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3471	}
3472
3473	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3474	}
3475
3476
3477	/**
3478	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3479	*
3480	* @param pVCpu The cross context virtual CPU structure.
3481	*/
3482	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPU pVCpu)
3483	{
3484	SVMEVENT Event;
3485	Event.u = 0;
3486	Event.n.u1Valid = 1;
3487	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3488	Event.n.u8Vector = X86_XCPT_MF;
3489	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3490	}
3491
3492
3493	/**
3494	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3495	*
3496	* @param pVCpu The cross context virtual CPU structure.
3497	*/
3498	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPU pVCpu)
3499	{
3500	SVMEVENT Event;
3501	Event.u = 0;
3502	Event.n.u1Valid = 1;
3503	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3504	Event.n.u8Vector = X86_XCPT_DF;
3505	Event.n.u1ErrorCodeValid = 1;
3506	Event.n.u32ErrorCode = 0;
3507	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3508	}
3509
3510
3511	/**
3512	* Injects an event into the guest upon VMRUN by updating the relevant field
3513	* in the VMCB.
3514	*
3515	* @param pVCpu The cross context virtual CPU structure.
3516	* @param pVmcb Pointer to the guest VM control block.
3517	* @param pEvent Pointer to the event.
3518	*
3519	* @remarks No-long-jump zone!!!
3520	* @remarks Requires CR0!
3521	*/
3522	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPU pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3523	{
3524	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3525	pVmcb->ctrl.EventInject.u = pEvent->u;
3526	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3527	RT_NOREF(pVCpu);
3528
3529	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3530	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3531	}
3532
3533
3534
3535	/**
3536	* Converts any TRPM trap into a pending HM event. This is typically used when
3537	* entering from ring-3 (not longjmp returns).
3538	*
3539	* @param pVCpu The cross context virtual CPU structure.
3540	*/
3541	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPU pVCpu)
3542	{
3543	Assert(TRPMHasTrap(pVCpu));
3544	Assert(!pVCpu->hm.s.Event.fPending);
3545
3546	uint8_t uVector;
3547	TRPMEVENT enmTrpmEvent;
3548	RTGCUINT uErrCode;
3549	RTGCUINTPTR GCPtrFaultAddress;
3550	uint8_t cbInstr;
3551
3552	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
3553	AssertRC(rc);
3554
3555	SVMEVENT Event;
3556	Event.u = 0;
3557	Event.n.u1Valid = 1;
3558	Event.n.u8Vector = uVector;
3559
3560	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3561	if (enmTrpmEvent == TRPM_TRAP)
3562	{
3563	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3564	switch (uVector)
3565	{
3566	case X86_XCPT_NMI:
3567	{
3568	Event.n.u3Type = SVM_EVENT_NMI;
3569	break;
3570	}
3571
3572	case X86_XCPT_PF:
3573	case X86_XCPT_DF:
3574	case X86_XCPT_TS:
3575	case X86_XCPT_NP:
3576	case X86_XCPT_SS:
3577	case X86_XCPT_GP:
3578	case X86_XCPT_AC:
3579	{
3580	Event.n.u1ErrorCodeValid = 1;
3581	Event.n.u32ErrorCode = uErrCode;
3582	break;
3583	}
3584	}
3585	}
3586	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3587	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3588	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3589	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3590	else
3591	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3592
3593	rc = TRPMResetTrap(pVCpu);
3594	AssertRC(rc);
3595
3596	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3597	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3598
3599	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3600	}
3601
3602
3603	/**
3604	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3605	* AMD-V to execute any instruction.
3606	*
3607	* @param pVCpu The cross context virtual CPU structure.
3608	*/
3609	static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu)
3610	{
3611	Assert(pVCpu->hm.s.Event.fPending);
3612	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3613
3614	SVMEVENT Event;
3615	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3616
3617	uint8_t uVector = Event.n.u8Vector;
3618	uint8_t uVectorType = Event.n.u3Type;
3619	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event);
3620
3621	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, uVectorType));
3622
3623	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3624	AssertRC(rc);
3625
3626	if (Event.n.u1ErrorCodeValid)
3627	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3628
3629	if ( uVectorType == SVM_EVENT_EXCEPTION
3630	&& uVector == X86_XCPT_PF)
3631	{
3632	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3633	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3634	}
3635	else if (uVectorType == SVM_EVENT_SOFTWARE_INT)
3636	{
3637	AssertMsg( uVectorType == SVM_EVENT_SOFTWARE_INT
3638	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
3639	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
3640	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3641	}
3642	pVCpu->hm.s.Event.fPending = false;
3643	}
3644
3645
3646	/**
3647	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3648	* now.
3649	*
3650	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3651	* @param pVCpu The cross context virtual CPU structure.
3652	* @param pCtx Pointer to the guest-CPU context.
3653	*
3654	* @remarks No-long-jump zone!!!
3655	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3656	*/
3657	DECLINLINE(bool) hmR0SvmIsIntrShadowActive(PVMCPU pVCpu, PCCPUMCTX pCtx)
3658	{
3659	/*
3660	* Instructions like STI and MOV SS inhibit interrupts till the next instruction
3661	* completes. Check if we should inhibit interrupts or clear any existing
3662	* interrupt inhibition.
3663	*/
3664	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3665	{
3666	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3667	{
3668	/*
3669	* We can clear the inhibit force flag as even if we go back to the recompiler
3670	* without executing guest code in AMD-V, the flag's condition to be cleared is
3671	* met and thus the cleared state is correct.
3672	*/
3673	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3674	return false;
3675	}
3676	return true;
3677	}
3678	return false;
3679	}
3680
3681
3682	/**
3683	* Sets the virtual interrupt intercept control in the VMCB.
3684	*
3685	* @param pVCpu The cross context virtual CPU structure.
3686	* @param pVmcb Pointer to the VM control block.
3687	* @param pCtx Pointer to the guest-CPU context.
3688	*/
3689	DECLINLINE(void) hmR0SvmSetIntWindowExiting(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
3690	{
3691	/*
3692	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3693	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3694	* (updating ISR at the right time) and inject the interrupt.
3695	*
3696	* With AVIC is supported, we could make use of the asynchronously delivery without
3697	* #VMEXIT and we would be passing the AVIC page to SVM.
3698	*
3699	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3700	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3701	*/
3702	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3703	/*
3704	* Currently we don't overlay interupt windows and if there's any V_IRQ pending in the
3705	* nested-guest VMCB, we avoid setting up any interrupt window on behalf of the outer
3706	* guest.
3707	*/
3708	/** @todo Does this mean we end up prioritizing virtual interrupt
3709	* delivery/window over a physical interrupt (from the outer guest)
3710	* might be pending? */
3711	bool const fEnableIntWindow = !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
3712	if (!fEnableIntWindow)
3713	{
3714	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); RT_NOREF(pCtx);
3715	Log4(("Nested-guest V_IRQ already pending\n"));
3716	}
3717	#else
3718	RT_NOREF2(pVCpu, pCtx);
3719	bool const fEnableIntWindow = true;
3720	#endif
3721	if (fEnableIntWindow)
3722	{
3723	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3724	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3725	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3726	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3727	Log4(("Set VINTR intercept\n"));
3728	}
3729	}
3730
3731
3732	/**
3733	* Clears the virtual interrupt intercept control in the VMCB as
3734	* we are figured the guest is unable process any interrupts
3735	* at this point of time.
3736	*
3737	* @param pVCpu The cross context virtual CPU structure.
3738	* @param pVmcb Pointer to the VM control block.
3739	* @param pCtx Pointer to the guest-CPU context.
3740	*/
3741	DECLINLINE(void) hmR0SvmClearIntWindowExiting(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
3742	{
3743	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3744	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3745	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3746	{
3747	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3748	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3749	hmR0SvmClearCtrlIntercept(pVCpu, pCtx, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3750	Log4(("Cleared VINTR intercept\n"));
3751	}
3752	}
3753
3754	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3755	/**
3756	* Evaluates the event to be delivered to the nested-guest and sets it as the
3757	* pending event.
3758	*
3759	* @returns VBox strict status code.
3760	* @param pVCpu The cross context virtual CPU structure.
3761	* @param pCtx Pointer to the guest-CPU context.
3762	*/
3763	static VBOXSTRICTRC hmR0SvmEvaluatePendingEventNested(PVMCPU pVCpu, PCPUMCTX pCtx)
3764	{
3765	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
3766	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3767	\| CPUMCTX_EXTRN_RFLAGS
3768	\| CPUMCTX_EXTRN_HM_SVM_INT_SHADOW
3769	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3770
3771	Assert(!pVCpu->hm.s.Event.fPending);
3772	Assert(pCtx->hwvirt.fGif);
3773	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
3774	Assert(pVmcb);
3775
3776	bool const fVirtualGif = CPUMGetSvmNstGstVGif(pCtx);
3777	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3778	bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3779
3780	Log4Func(("fVirtualGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3781	fVirtualGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3782	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3783
3784	/** @todo SMI. SMIs take priority over NMIs. */
3785
3786	/*
3787	* Check if the guest can receive NMIs.
3788	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3789	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3790	*/
3791	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3792	&& !fBlockNmi)
3793	{
3794	if ( fVirtualGif
3795	&& !fIntShadow)
3796	{
3797	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3798	{
3799	Log4(("Intercepting NMI -> #VMEXIT\n"));
3800	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
3801	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3802	}
3803
3804	Log4(("Setting NMI pending for injection\n"));
3805	SVMEVENT Event;
3806	Event.u = 0;
3807	Event.n.u1Valid = 1;
3808	Event.n.u8Vector = X86_XCPT_NMI;
3809	Event.n.u3Type = SVM_EVENT_NMI;
3810	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3811	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3812	}
3813	else if (!fVirtualGif)
3814	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3815	else
3816	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb, pCtx);
3817	}
3818	/*
3819	* Check if the nested-guest can receive external interrupts (generated by the guest's
3820	* PIC/APIC).
3821	*
3822	* External intercepts, NMI, SMI etc. from the physical CPU are -always- intercepted
3823	* when executing using hardware-assisted SVM, see HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS.
3824	*
3825	* External interrupts that are generated for the outer guest may be intercepted
3826	* depending on how the nested-guest VMCB was programmed by guest software.
3827	*
3828	* Physical interrupts always take priority over virtual interrupts,
3829	* see AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3830	*/
3831	else if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3832	&& !pVCpu->hm.s.fSingleInstruction)
3833	{
3834	if ( fVirtualGif
3835	&& !fIntShadow
3836	&& CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx))
3837	{
3838	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3839	{
3840	Log4(("Intercepting INTR -> #VMEXIT\n"));
3841	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
3842	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3843	}
3844
3845	uint8_t u8Interrupt;
3846	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3847	if (RT_SUCCESS(rc))
3848	{
3849	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3850	SVMEVENT Event;
3851	Event.u = 0;
3852	Event.n.u1Valid = 1;
3853	Event.n.u8Vector = u8Interrupt;
3854	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3855	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3856	}
3857	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3858	{
3859	/*
3860	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3861	* updated eventually when the TPR is written by the guest.
3862	*/
3863	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3864	}
3865	else
3866	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3867	}
3868	else if (!fVirtualGif)
3869	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3870	else
3871	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb, pCtx);
3872	}
3873
3874	return VINF_SUCCESS;
3875	}
3876	#endif
3877
3878	/**
3879	* Evaluates the event to be delivered to the guest and sets it as the pending
3880	* event.
3881	*
3882	* @param pVCpu The cross context virtual CPU structure.
3883	* @param pCtx Pointer to the guest-CPU context.
3884	*/
3885	static void hmR0SvmEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
3886	{
3887	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3888	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3889	\| CPUMCTX_EXTRN_RFLAGS
3890	\| CPUMCTX_EXTRN_HM_SVM_INT_SHADOW);
3891
3892	Assert(!pVCpu->hm.s.Event.fPending);
3893	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
3894	Assert(pVmcb);
3895
3896	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3897	bool const fGif = pCtx->hwvirt.fGif;
3898	#else
3899	bool const fGif = true;
3900	#endif
3901	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3902	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
3903	bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3904
3905	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fBlockInt=%RTbool fIntShadow=%RTbool fIntPending=%RTbool NMI pending=%RTbool\n",
3906	fGif, fBlockNmi, fBlockInt, fIntShadow,
3907	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3908	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3909
3910	/** @todo SMI. SMIs take priority over NMIs. */
3911
3912	/*
3913	* Check if the guest can receive NMIs.
3914	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3915	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3916	*/
3917	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3918	&& !fBlockNmi)
3919	{
3920	if ( fGif
3921	&& !fIntShadow)
3922	{
3923	Log4(("Setting NMI pending for injection\n"));
3924	SVMEVENT Event;
3925	Event.u = 0;
3926	Event.n.u1Valid = 1;
3927	Event.n.u8Vector = X86_XCPT_NMI;
3928	Event.n.u3Type = SVM_EVENT_NMI;
3929	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3930	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3931	}
3932	else if (!fGif)
3933	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3934	else
3935	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb, pCtx);
3936	}
3937	/*
3938	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3939	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3940	* it from the APIC device.
3941	*/
3942	else if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3943	&& !pVCpu->hm.s.fSingleInstruction)
3944	{
3945	if ( fGif
3946	&& !fBlockInt
3947	&& !fIntShadow)
3948	{
3949	uint8_t u8Interrupt;
3950	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3951	if (RT_SUCCESS(rc))
3952	{
3953	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3954	SVMEVENT Event;
3955	Event.u = 0;
3956	Event.n.u1Valid = 1;
3957	Event.n.u8Vector = u8Interrupt;
3958	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3959	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3960	}
3961	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3962	{
3963	/*
3964	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3965	* updated eventually when the TPR is written by the guest.
3966	*/
3967	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3968	}
3969	else
3970	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3971	}
3972	else if (!fGif)
3973	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3974	else
3975	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb, pCtx);
3976	}
3977	}
3978
3979
3980	/**
3981	* Injects any pending events into the guest (or nested-guest).
3982	*
3983	* @param pVCpu The cross context virtual CPU structure.
3984	* @param pCtx Pointer to the guest-CPU context.
3985	* @param pVmcb Pointer to the VM control block.
3986	*
3987	* @remarks Must only be called when we are guaranteed to enter
3988	* hardware-assisted SVM execution and not return to ring-3
3989	* prematurely.
3990	*/
3991	static void hmR0SvmInjectPendingEvent(PVMCPU pVCpu, PCCPUMCTX pCtx, PSVMVMCB pVmcb)
3992	{
3993	Assert(!TRPMHasTrap(pVCpu));
3994	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3995
3996	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3997	#ifdef VBOX_STRICT
3998	bool const fGif = pCtx->hwvirt.fGif;
3999	bool fAllowInt = fGif;
4000	if (fGif)
4001	{
4002	/*
4003	* For nested-guests we have no way to determine if we're injecting a physical or
4004	* virtual interrupt at this point. Hence the partial verification below.
4005	*/
4006	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4007	fAllowInt = CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx) \|\| CPUMCanSvmNstGstTakeVirtIntr(pVCpu, pCtx);
4008	else
4009	fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
4010	}
4011	#endif
4012
4013	if (pVCpu->hm.s.Event.fPending)
4014	{
4015	SVMEVENT Event;
4016	Event.u = pVCpu->hm.s.Event.u64IntInfo;
4017	Assert(Event.n.u1Valid);
4018
4019	/*
4020	* Validate event injection pre-conditions.
4021	*/
4022	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
4023	{
4024	Assert(fAllowInt);
4025	Assert(!fIntShadow);
4026	}
4027	else if (Event.n.u3Type == SVM_EVENT_NMI)
4028	{
4029	Assert(fGif);
4030	Assert(!fIntShadow);
4031	}
4032
4033	/*
4034	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
4035	* do this only when we are surely going to inject the NMI as otherwise if we return
4036	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
4037	* SVM R0.
4038	*
4039	* With VT-x, this is handled by the Guest interruptibility information VMCS field
4040	* which will set the VMCS field after actually delivering the NMI which we read on
4041	* VM-exit to determine the state.
4042	*/
4043	if ( Event.n.u3Type == SVM_EVENT_NMI
4044	&& Event.n.u8Vector == X86_XCPT_NMI
4045	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
4046	{
4047	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
4048	}
4049
4050	/*
4051	* Inject it (update VMCB for injection by the hardware).
4052	*/
4053	Log4(("Injecting pending HM event\n"));
4054	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
4055	pVCpu->hm.s.Event.fPending = false;
4056
4057	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
4058	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
4059	else
4060	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
4061	}
4062	else
4063	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
4064
4065	/*
4066	* We could have injected an NMI through IEM and continue guest execution using
4067	* hardware-assisted SVM. In which case, we would not have any events pending (above)
4068	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
4069	*/
4070	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
4071	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
4072
4073	/*
4074	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
4075	*
4076	* For nested-guests: We need to update it too for the scenario where IEM executes
4077	* the nested-guest but execution later continues here with an interrupt shadow active.
4078	*/
4079	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
4080	}
4081
4082
4083	/**
4084	* Reports world-switch error and dumps some useful debug info.
4085	*
4086	* @param pVM The cross context VM structure.
4087	* @param pVCpu The cross context virtual CPU structure.
4088	* @param rcVMRun The return code from VMRUN (or
4089	* VERR_SVM_INVALID_GUEST_STATE for invalid
4090	* guest-state).
4091	* @param pCtx Pointer to the guest-CPU context.
4092	*/
4093	static void hmR0SvmReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx)
4094	{
4095	NOREF(pCtx);
4096	HMSVM_ASSERT_PREEMPT_SAFE();
4097	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
4098	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4099
4100	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4101	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
4102	{
4103	hmR0DumpRegs(pVM, pVCpu, pCtx); NOREF(pVM);
4104	/** @todo We probably don't need to dump this anymore or we can expand
4105	* hmR0DumpRegs()? */
4106	#ifdef VBOX_STRICT
4107	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
4108	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
4109	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
4110	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
4111	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
4112	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
4113	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
4114	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
4115	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
4116	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
4117
4118	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
4119	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
4120	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
4121
4122	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
4123	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
4124	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
4125	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
4126	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
4127	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
4128	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
4129	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
4130	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
4131	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
4132	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
4133	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
4134
4135	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
4136	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
4137	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
4138	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
4139	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
4140	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
4141	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
4142	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
4143	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
4144	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
4145	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
4146	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
4147	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
4148	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
4149	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
4150	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
4151	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
4152	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
4153	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
4154	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
4155
4156	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
4157
4158	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
4159	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
4160
4161	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
4162	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
4163	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
4164	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
4165	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
4166	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
4167	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
4168	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
4169	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
4170	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
4171	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
4172	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
4173	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
4174	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
4175	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
4176	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
4177	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
4178	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
4179	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
4180	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
4181
4182	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
4183	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
4184
4185	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
4186	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
4187	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
4188	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
4189
4190	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
4191	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
4192
4193	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
4194	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
4195	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
4196	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
4197
4198	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
4199	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
4200	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
4201	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
4202	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
4203	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
4204	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
4205
4206	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
4207	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
4208	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
4209	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
4210
4211	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
4212	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
4213	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
4214
4215	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
4216	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
4217	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
4218	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
4219	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
4220	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
4221	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
4222	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
4223	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
4224	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
4225	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
4226	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
4227	#endif /* VBOX_STRICT */
4228	}
4229	else
4230	Log4Func(("rcVMRun=%d\n", rcVMRun));
4231
4232	NOREF(pVmcb);
4233	}
4234
4235
4236	/**
4237	* Check per-VM and per-VCPU force flag actions that require us to go back to
4238	* ring-3 for one reason or another.
4239	*
4240	* @returns VBox status code (information status code included).
4241	* @retval VINF_SUCCESS if we don't have any actions that require going back to
4242	* ring-3.
4243	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
4244	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
4245	* interrupts)
4246	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
4247	* all EMTs to be in ring-3.
4248	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
4249	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
4250	* to the EM loop.
4251	*
4252	* @param pVM The cross context VM structure.
4253	* @param pVCpu The cross context virtual CPU structure.
4254	* @param pCtx Pointer to the guest-CPU context.
4255	*/
4256	static int hmR0SvmCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4257	{
4258	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4259	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
4260
4261	/* Could happen as a result of longjump. */
4262	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
4263	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
4264
4265	/* Update pending interrupts into the APIC's IRR. */
4266	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
4267	APICUpdatePendingInterrupts(pVCpu);
4268
4269	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
4270	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
4271	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
4272	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
4273	{
4274	/* Pending PGM C3 sync. */
4275	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
4276	{
4277	int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4278	if (rc != VINF_SUCCESS)
4279	{
4280	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
4281	return rc;
4282	}
4283	}
4284
4285	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4286	/* -XXX- what was that about single stepping? */
4287	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
4288	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4289	{
4290	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4291	int rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
4292	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
4293	return rc;
4294	}
4295
4296	/* Pending VM request packets, such as hardware interrupts. */
4297	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
4298	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
4299	{
4300	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4301	return VINF_EM_PENDING_REQUEST;
4302	}
4303
4304	/* Pending PGM pool flushes. */
4305	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4306	{
4307	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4308	return VINF_PGM_POOL_FLUSH_PENDING;
4309	}
4310
4311	/* Pending DMA requests. */
4312	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
4313	{
4314	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4315	return VINF_EM_RAW_TO_R3;
4316	}
4317	}
4318
4319	return VINF_SUCCESS;
4320	}
4321
4322
4323	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4324	/**
4325	* Does the preparations before executing nested-guest code in AMD-V.
4326	*
4327	* @returns VBox status code (informational status codes included).
4328	* @retval VINF_SUCCESS if we can proceed with running the guest.
4329	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4330	*
4331	* @param pVM The cross context VM structure.
4332	* @param pVCpu The cross context virtual CPU structure.
4333	* @param pCtx Pointer to the nested-guest-CPU context.
4334	* @param pSvmTransient Pointer to the SVM transient structure.
4335	*
4336	* @remarks Same caveats regarding longjumps as hmR0SvmPreRunGuest applies.
4337	* @sa hmR0SvmPreRunGuest.
4338	*/
4339	static int hmR0SvmPreRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4340	{
4341	HMSVM_ASSERT_PREEMPT_SAFE();
4342	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4343
4344	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM_ONLY_IN_IEM
4345	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4346	return VINF_EM_RESCHEDULE_REM;
4347	#endif
4348
4349	/* Check force flag actions that might require us to go back to ring-3. */
4350	int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
4351	if (rc != VINF_SUCCESS)
4352	return rc;
4353
4354	if (TRPMHasTrap(pVCpu))
4355	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4356	else if (!pVCpu->hm.s.Event.fPending)
4357	{
4358	VBOXSTRICTRC rcStrict = hmR0SvmEvaluatePendingEventNested(pVCpu, pCtx);
4359	if ( rcStrict != VINF_SUCCESS
4360	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4361	return VBOXSTRICTRC_VAL(rcStrict);
4362	}
4363
4364	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4365
4366	/*
4367	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4368	* Just do it in software, see @bugref{8411}.
4369	* NB: If we could continue a task switch exit we wouldn't need to do this.
4370	*/
4371	if (RT_UNLIKELY( !pVM->hm.s.svm.u32Features
4372	&& pVCpu->hm.s.Event.fPending
4373	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4374	{
4375	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4376	}
4377
4378	#ifdef HMSVM_SYNC_FULL_NESTED_GUEST_STATE
4379	Assert(!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4380	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4381	#endif
4382
4383	/*
4384	* Export the nested-guest state bits that are not shared with the host in any way as we
4385	* can longjmp or get preempted in the midst of exporting some of the state.
4386	*/
4387	rc = hmR0SvmExportGuestStateNested(pVCpu);
4388	AssertRCReturn(rc, rc);
4389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4390
4391	/* Ensure we've cached (and hopefully modified) the VMCB for execution using hardware-assisted SVM. */
4392	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4393
4394	/*
4395	* No longjmps to ring-3 from this point on!!!
4396	*
4397	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4398	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4399	*/
4400	VMMRZCallRing3Disable(pVCpu);
4401
4402	/*
4403	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4404	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4405	* preemption disabled for a while. Since this is purly to aid the
4406	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4407	* disable interrupt on NT.
4408	*
4409	* We need to check for force-flags that could've possible been altered since we last
4410	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4411	* see @bugref{6398}).
4412	*
4413	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4414	* to ring-3 before executing guest code.
4415	*/
4416	pSvmTransient->fEFlags = ASMIntDisableFlags();
4417	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4418	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4419	{
4420	ASMSetFlags(pSvmTransient->fEFlags);
4421	VMMRZCallRing3Enable(pVCpu);
4422	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4423	return VINF_EM_RAW_TO_R3;
4424	}
4425	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4426	{
4427	ASMSetFlags(pSvmTransient->fEFlags);
4428	VMMRZCallRing3Enable(pVCpu);
4429	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
4430	return VINF_EM_RAW_INTERRUPT;
4431	}
4432	return VINF_SUCCESS;
4433	}
4434	#endif
4435
4436
4437	/**
4438	* Does the preparations before executing guest code in AMD-V.
4439	*
4440	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4441	* recompiler. We must be cautious what we do here regarding committing
4442	* guest-state information into the VMCB assuming we assuredly execute the guest
4443	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4444	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4445	* that the recompiler can (and should) use them when it resumes guest
4446	* execution. Otherwise such operations must be done when we can no longer
4447	* exit to ring-3.
4448	*
4449	* @returns VBox status code (informational status codes included).
4450	* @retval VINF_SUCCESS if we can proceed with running the guest.
4451	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4452	*
4453	* @param pVM The cross context VM structure.
4454	* @param pVCpu The cross context virtual CPU structure.
4455	* @param pCtx Pointer to the guest-CPU context.
4456	* @param pSvmTransient Pointer to the SVM transient structure.
4457	*/
4458	static int hmR0SvmPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4459	{
4460	HMSVM_ASSERT_PREEMPT_SAFE();
4461	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
4462
4463	/* Check force flag actions that might require us to go back to ring-3. */
4464	int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
4465	if (rc != VINF_SUCCESS)
4466	return rc;
4467
4468	if (TRPMHasTrap(pVCpu))
4469	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4470	else if (!pVCpu->hm.s.Event.fPending)
4471	hmR0SvmEvaluatePendingEvent(pVCpu, pCtx);
4472
4473	/*
4474	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4475	* Just do it in software, see @bugref{8411}.
4476	* NB: If we could continue a task switch exit we wouldn't need to do this.
4477	*/
4478	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending && (((pVCpu->hm.s.Event.u64IntInfo >> 8) & 7) == SVM_EVENT_NMI)))
4479	if (RT_UNLIKELY(!pVM->hm.s.svm.u32Features))
4480	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4481
4482	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4483	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4484	{
4485	Assert(!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4486	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4487	}
4488	#endif
4489	#ifdef HMSVM_SYNC_FULL_NESTED_GUEST_STATE
4490	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4491	{
4492	Assert(!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4494	}
4495	#endif
4496
4497	/*
4498	* Export the guest state bits that are not shared with the host in any way as we can
4499	* longjmp or get preempted in the midst of exporting some of the state.
4500	*/
4501	rc = hmR0SvmExportGuestState(pVCpu);
4502	AssertRCReturn(rc, rc);
4503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4504
4505	/*
4506	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4507	* world-switch so we can update it on the way back if the guest changed the TPR.
4508	*/
4509	if (pVCpu->hm.s.svm.fSyncVTpr)
4510	{
4511	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4512	if (pVM->hm.s.fTPRPatchingActive)
4513	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4514	else
4515	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4516	}
4517
4518	/*
4519	* No longjmps to ring-3 from this point on!!!
4520	*
4521	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4522	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4523	*/
4524	VMMRZCallRing3Disable(pVCpu);
4525
4526	/*
4527	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4528	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4529	* preemption disabled for a while. Since this is purly to aid the
4530	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4531	* disable interrupt on NT.
4532	*
4533	* We need to check for force-flags that could've possible been altered since we last
4534	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4535	* see @bugref{6398}).
4536	*
4537	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4538	* to ring-3 before executing guest code.
4539	*/
4540	pSvmTransient->fEFlags = ASMIntDisableFlags();
4541	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4542	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4543	{
4544	ASMSetFlags(pSvmTransient->fEFlags);
4545	VMMRZCallRing3Enable(pVCpu);
4546	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4547	return VINF_EM_RAW_TO_R3;
4548	}
4549	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4550	{
4551	ASMSetFlags(pSvmTransient->fEFlags);
4552	VMMRZCallRing3Enable(pVCpu);
4553	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
4554	return VINF_EM_RAW_INTERRUPT;
4555	}
4556
4557	return VINF_SUCCESS;
4558	}
4559
4560
4561	/**
4562	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4563	* doing so.
4564	*
4565	* This means there is no backing out to ring-3 or anywhere else at this point.
4566	*
4567	* @param pVCpu The cross context virtual CPU structure.
4568	* @param pCtx Pointer to the guest-CPU context.
4569	* @param pSvmTransient Pointer to the SVM transient structure.
4570	*
4571	* @remarks Called with preemption disabled.
4572	* @remarks No-long-jump zone!!!
4573	*/
4574	static void hmR0SvmPreRunGuestCommitted(PVMCPU pVCpu, PCCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4575	{
4576	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4577	Assert(VMMR0IsLogFlushDisabled(pVCpu));
4578	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4579
4580	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4581	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4582
4583	PVM pVM = pVCpu->CTX_SUFF(pVM);
4584	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4585
4586	hmR0SvmInjectPendingEvent(pVCpu, pCtx, pVmcb);
4587
4588	if (!CPUMIsGuestFPUStateActive(pVCpu))
4589	{
4590	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4591	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4592	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4593	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4594	}
4595
4596	/* Load the state shared between host and guest (FPU, debug). */
4597	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4598	hmR0SvmExportSharedState(pVCpu, pVmcb);
4599
4600	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4601	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4602
4603	PHMGLOBALCPUINFO pHostCpu = hmR0GetCurrentCpu();
4604	RTCPUID const idHostCpu = pHostCpu->idCpu;
4605	bool const fMigratedHostCpu = idHostCpu != pVCpu->hm.s.idLastCpu;
4606
4607	/* Setup TSC offsetting. */
4608	if ( pSvmTransient->fUpdateTscOffsetting
4609	\|\| fMigratedHostCpu)
4610	{
4611	hmR0SvmUpdateTscOffsetting(pVM, pVCpu, pCtx, pVmcb);
4612	pSvmTransient->fUpdateTscOffsetting = false;
4613	}
4614
4615	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4616	if (fMigratedHostCpu)
4617	pVmcb->ctrl.u32VmcbCleanBits = 0;
4618
4619	/* Store status of the shared guest-host state at the time of VMRUN. */
4620	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4621	if (CPUMIsGuestInLongModeEx(pCtx))
4622	{
4623	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4624	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4625	}
4626	else
4627	#endif
4628	{
4629	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4630	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4631	}
4632
4633	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4634	uint8_t *pbMsrBitmap;
4635	if (!pSvmTransient->fIsNestedGuest)
4636	pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
4637	else
4638	{
4639	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu, pCtx);
4640
4641	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4642	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4643	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4644	}
4645	#else
4646	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4647	#endif
4648
4649	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4650	/* Flush the appropriate tagged-TLB entries. */
4651	hmR0SvmFlushTaggedTlb(pVCpu, pCtx, pVmcb, pHostCpu);
4652	Assert(pVCpu->hm.s.idLastCpu == idHostCpu);
4653
4654	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4655
4656	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4657	to start executing. */
4658
4659	/*
4660	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4661	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4662	*
4663	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4664	*/
4665	if ( (pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
4666	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4667	{
4668	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4669	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4670	if (uGuestTscAux != pVCpu->hm.s.u64HostTscAux)
4671	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4672	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4673	pSvmTransient->fRestoreTscAuxMsr = true;
4674	}
4675	else
4676	{
4677	hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4678	pSvmTransient->fRestoreTscAuxMsr = false;
4679	}
4680	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4681
4682	/*
4683	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4684	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4685	* from the VMCB.
4686	*/
4687	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pCtx);
4688	if (!fSupportsVmcbCleanBits)
4689	pVmcb->ctrl.u32VmcbCleanBits = 0;
4690	}
4691
4692
4693	/**
4694	* Wrapper for running the guest code in AMD-V.
4695	*
4696	* @returns VBox strict status code.
4697	* @param pVM The cross context VM structure.
4698	* @param pVCpu The cross context virtual CPU structure.
4699	* @param pCtx Pointer to the guest-CPU context.
4700	*
4701	* @remarks No-long-jump zone!!!
4702	*/
4703	DECLINLINE(int) hmR0SvmRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4704	{
4705	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4706	pCtx->fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4707
4708	/*
4709	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4710	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4711	* callee-saved and thus the need for this XMM wrapper.
4712	*
4713	* Refer MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4714	*/
4715	#ifdef VBOX_WITH_KERNEL_USING_XMM
4716	return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4717	pVCpu->hm.s.svm.pfnVMRun);
4718	#else
4719	return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4720	#endif
4721	}
4722
4723
4724	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4725	/**
4726	* Wrapper for running the nested-guest code in AMD-V.
4727	*
4728	* @returns VBox strict status code.
4729	* @param pVM The cross context VM structure.
4730	* @param pVCpu The cross context virtual CPU structure.
4731	* @param pCtx Pointer to the guest-CPU context.
4732	*
4733	* @remarks No-long-jump zone!!!
4734	*/
4735	DECLINLINE(int) hmR0SvmRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4736	{
4737	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4738	pCtx->fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4739
4740	/*
4741	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4742	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4743	* callee-saved and thus the need for this XMM wrapper.
4744	*
4745	* Refer MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4746	*/
4747	#ifdef VBOX_WITH_KERNEL_USING_XMM
4748	return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4749	pVCpu->hm.s.svm.pfnVMRun);
4750	#else
4751	return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4752	#endif
4753	}
4754
4755
4756	/**
4757	* Undoes the TSC offset applied for an SVM nested-guest and returns the TSC
4758	* value for the guest.
4759	*
4760	* @returns The TSC offset after undoing any nested-guest TSC offset.
4761	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4762	* @param uTicks The nested-guest TSC.
4763	*
4764	* @note If you make any changes to this function, please check if
4765	* hmR0SvmNstGstUndoTscOffset() needs adjusting.
4766	*
4767	* @sa HMSvmNstGstApplyTscOffset().
4768	*/
4769	DECLINLINE(uint64_t) hmR0SvmNstGstUndoTscOffset(PVMCPU pVCpu, uint64_t uTicks)
4770	{
4771	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
4772	Assert(pVmcbNstGstCache->fCacheValid);
4773	return uTicks - pVmcbNstGstCache->u64TSCOffset;
4774	}
4775	#endif
4776
4777	/**
4778	* Performs some essential restoration of state after running guest (or
4779	* nested-guest) code in AMD-V.
4780	*
4781	* @param pVCpu The cross context virtual CPU structure.
4782	* @param pCtx Pointer to the guest-CPU context. The data maybe
4783	* out-of-sync. Make sure to update the required fields
4784	* before using them.
4785	* @param pSvmTransient Pointer to the SVM transient structure.
4786	* @param rcVMRun Return code of VMRUN.
4787	*
4788	* @remarks Called with interrupts disabled.
4789	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4790	* unconditionally when it is safe to do so.
4791	*/
4792	static void hmR0SvmPostRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4793	{
4794	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4795
4796	uint64_t const uHostTsc = ASMReadTSC(); /* Read the TSC as soon as possible. */
4797	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4798	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4799
4800	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4801	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4802
4803	/* TSC read must be done early for maximum accuracy. */
4804	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4805	{
4806	if (!pSvmTransient->fIsNestedGuest)
4807	TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
4808	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4809	else
4810	{
4811	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMSvmNstGstVmExitNotify(). */
4812	uint64_t const uGstTsc = hmR0SvmNstGstUndoTscOffset(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
4813	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4814	}
4815	#endif
4816	}
4817
4818	if (pSvmTransient->fRestoreTscAuxMsr)
4819	{
4820	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4821	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4822	if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4823	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4824	}
4825
4826	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
4827	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4828	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4829
4830	Assert(!(ASMGetFlags() & X86_EFL_IF));
4831	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4832	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4833
4834	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4835	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4836	{
4837	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4838	return;
4839	}
4840
4841	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4842	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4843	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4844	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4845
4846	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4847	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4848	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4849	#elif defined(HMSVM_SYNC_FULL_NESTED_GUEST_STATE)
4850	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4851	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4852	#else
4853	/*
4854	* Always import the following:
4855	*
4856	* - RIP for exit optimizations and evaluating event injection on re-entry.
4857	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4858	* state on preemption.
4859	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4860	* - CS for exit optimizations.
4861	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4862	* assembly switcher code.
4863	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4864	*/
4865	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4866	\| CPUMCTX_EXTRN_RFLAGS
4867	\| CPUMCTX_EXTRN_RAX
4868	\| CPUMCTX_EXTRN_RSP
4869	\| CPUMCTX_EXTRN_CS
4870	\| CPUMCTX_EXTRN_HWVIRT
4871	\| CPUMCTX_EXTRN_HM_SVM_INT_SHADOW
4872	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4873	\| HMSVM_CPUMCTX_SHARED_STATE);
4874	#endif
4875
4876	#ifdef DEBUG_ramshankar
4877	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4878	{
4879	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4880	hmR0SvmLogState(pVCpu, pVmcb, pCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR, 0 /* uVerbose */);
4881	}
4882	#endif
4883
4884	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4885	&& pVCpu->hm.s.svm.fSyncVTpr)
4886	{
4887	Assert(!pSvmTransient->fIsNestedGuest);
4888	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4889	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive
4890	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4891	{
4892	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4893	AssertRC(rc);
4894	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4895	}
4896	/* Sync TPR when we aren't intercepting CR8 writes. */
4897	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4898	{
4899	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4900	AssertRC(rc);
4901	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4902	}
4903	}
4904
4905	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4906	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4907	pCtx->cs.u64Base + pCtx->rip, uHostTsc);
4908	}
4909
4910
4911	/**
4912	* Runs the guest code using AMD-V.
4913	*
4914	* @returns VBox status code.
4915	* @param pVM The cross context VM structure.
4916	* @param pVCpu The cross context virtual CPU structure.
4917	* @param pCtx Pointer to the guest-CPU context.
4918	* @param pcLoops Pointer to the number of executed loops.
4919	*/
4920	static int hmR0SvmRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4921	{
4922	uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4923	Assert(pcLoops);
4924	Assert(*pcLoops <= cMaxResumeLoops);
4925
4926	SVMTRANSIENT SvmTransient;
4927	RT_ZERO(SvmTransient);
4928	SvmTransient.fUpdateTscOffsetting = true;
4929	SvmTransient.pVmcb = pVCpu->hm.s.svm.pVmcb;
4930
4931	int rc = VERR_INTERNAL_ERROR_5;
4932	for (;;)
4933	{
4934	Assert(!HMR0SuspendPending());
4935	HMSVM_ASSERT_CPU_SAFE();
4936
4937	/* Preparatory work for running nested-guest code, this may force us to return to
4938	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4939	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4940	rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4941	if (rc != VINF_SUCCESS)
4942	break;
4943
4944	/*
4945	* No longjmps to ring-3 from this point on!!!
4946	*
4947	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4948	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4949	*/
4950	hmR0SvmPreRunGuestCommitted(pVCpu, pCtx, &SvmTransient);
4951	rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4952
4953	/* Restore any residual host-state and save any bits shared between host and guest
4954	into the guest-CPU state. Re-enables interrupts! */
4955	hmR0SvmPostRunGuest(pVCpu, pCtx, &SvmTransient, rc);
4956
4957	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4958	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4959	{
4960	if (rc == VINF_SUCCESS)
4961	rc = VERR_SVM_INVALID_GUEST_STATE;
4962	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4963	hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4964	break;
4965	}
4966
4967	/* Handle the #VMEXIT. */
4968	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4969	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4970	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4971	rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4972	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4973	if (rc != VINF_SUCCESS)
4974	break;
4975	if (++(*pcLoops) >= cMaxResumeLoops)
4976	{
4977	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4978	rc = VINF_EM_RAW_INTERRUPT;
4979	break;
4980	}
4981	}
4982
4983	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4984	return rc;
4985	}
4986
4987
4988	/**
4989	* Runs the guest code using AMD-V in single step mode.
4990	*
4991	* @returns VBox status code.
4992	* @param pVM The cross context VM structure.
4993	* @param pVCpu The cross context virtual CPU structure.
4994	* @param pCtx Pointer to the guest-CPU context.
4995	* @param pcLoops Pointer to the number of executed loops.
4996	*/
4997	static int hmR0SvmRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4998	{
4999	uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
5000	Assert(pcLoops);
5001	Assert(*pcLoops <= cMaxResumeLoops);
5002
5003	SVMTRANSIENT SvmTransient;
5004	RT_ZERO(SvmTransient);
5005	SvmTransient.fUpdateTscOffsetting = true;
5006	SvmTransient.pVmcb = pVCpu->hm.s.svm.pVmcb;
5007
5008	uint16_t uCsStart = pCtx->cs.Sel;
5009	uint64_t uRipStart = pCtx->rip;
5010
5011	int rc = VERR_INTERNAL_ERROR_5;
5012	for (;;)
5013	{
5014	Assert(!HMR0SuspendPending());
5015	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
5016	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
5017	(unsigned)RTMpCpuId(), *pcLoops));
5018
5019	/* Preparatory work for running nested-guest code, this may force us to return to
5020	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
5021	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
5022	rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
5023	if (rc != VINF_SUCCESS)
5024	break;
5025
5026	/*
5027	* No longjmps to ring-3 from this point on!!!
5028	*
5029	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
5030	* better than a kernel panic. This also disables flushing of the R0-logger instance.
5031	*/
5032	VMMRZCallRing3Disable(pVCpu);
5033	VMMRZCallRing3RemoveNotification(pVCpu);
5034	hmR0SvmPreRunGuestCommitted(pVCpu, pCtx, &SvmTransient);
5035
5036	rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
5037
5038	/* Restore any residual host-state and save any bits shared between host and guest
5039	into the guest-CPU state. Re-enables interrupts! */
5040	hmR0SvmPostRunGuest(pVCpu, pCtx, &SvmTransient, rc);
5041
5042	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
5043	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
5044	{
5045	if (rc == VINF_SUCCESS)
5046	rc = VERR_SVM_INVALID_GUEST_STATE;
5047	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5048	hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
5049	return rc;
5050	}
5051
5052	/* Handle the #VMEXIT. */
5053	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
5054	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
5055	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
5056	rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
5057	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5058	if (rc != VINF_SUCCESS)
5059	break;
5060	if (++(*pcLoops) >= cMaxResumeLoops)
5061	{
5062	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
5063	rc = VINF_EM_RAW_INTERRUPT;
5064	break;
5065	}
5066
5067	/*
5068	* Did the RIP change, if so, consider it a single step.
5069	* Otherwise, make sure one of the TFs gets set.
5070	*/
5071	if ( pCtx->rip != uRipStart
5072	\|\| pCtx->cs.Sel != uCsStart)
5073	{
5074	rc = VINF_EM_DBG_STEPPED;
5075	break;
5076	}
5077	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR_MASK;
5078	}
5079
5080	/*
5081	* Clear the X86_EFL_TF if necessary.
5082	*/
5083	if (pVCpu->hm.s.fClearTrapFlag)
5084	{
5085	pVCpu->hm.s.fClearTrapFlag = false;
5086	pCtx->eflags.Bits.u1TF = 0;
5087	}
5088
5089	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5090	return rc;
5091	}
5092
5093	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5094	/**
5095	* Runs the nested-guest code using AMD-V.
5096	*
5097	* @returns VBox status code.
5098	* @param pVM The cross context VM structure.
5099	* @param pVCpu The cross context virtual CPU structure.
5100	* @param pCtx Pointer to the guest-CPU context.
5101	* @param pcLoops Pointer to the number of executed loops. If we're switching
5102	* from the guest-code execution loop to this nested-guest
5103	* execution loop pass the remainder value, else pass 0.
5104	*/
5105	static int hmR0SvmRunGuestCodeNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
5106	{
5107	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
5108	Assert(pcLoops);
5109	Assert(*pcLoops <= pVM->hm.s.cMaxResumeLoops);
5110
5111	SVMTRANSIENT SvmTransient;
5112	RT_ZERO(SvmTransient);
5113	SvmTransient.fUpdateTscOffsetting = true;
5114	SvmTransient.pVmcb = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
5115	SvmTransient.fIsNestedGuest = true;
5116
5117	int rc = VERR_INTERNAL_ERROR_4;
5118	for (;;)
5119	{
5120	Assert(!HMR0SuspendPending());
5121	HMSVM_ASSERT_CPU_SAFE();
5122
5123	/* Preparatory work for running nested-guest code, this may force us to return to
5124	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
5125	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
5126	rc = hmR0SvmPreRunGuestNested(pVM, pVCpu, pCtx, &SvmTransient);
5127	if ( rc != VINF_SUCCESS
5128	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
5129	{
5130	break;
5131	}
5132
5133	/*
5134	* No longjmps to ring-3 from this point on!!!
5135	*
5136	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
5137	* better than a kernel panic. This also disables flushing of the R0-logger instance.
5138	*/
5139	hmR0SvmPreRunGuestCommitted(pVCpu, pCtx, &SvmTransient);
5140
5141	rc = hmR0SvmRunGuestNested(pVM, pVCpu, pCtx);
5142
5143	/* Restore any residual host-state and save any bits shared between host and guest
5144	into the guest-CPU state. Re-enables interrupts! */
5145	hmR0SvmPostRunGuest(pVCpu, pCtx, &SvmTransient, rc);
5146
5147	if (RT_LIKELY( rc == VINF_SUCCESS
5148	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
5149	{ /* extremely likely */ }
5150	else
5151	{
5152	/* VMRUN failed, shouldn't really happen, Guru. */
5153	if (rc != VINF_SUCCESS)
5154	break;
5155
5156	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
5157	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
5158	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
5159	rc = VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0));
5160	break;
5161	}
5162
5163	/* Handle the #VMEXIT. */
5164	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
5165	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
5166	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
5167	rc = hmR0SvmHandleExitNested(pVCpu, pCtx, &SvmTransient);
5168	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5169	if ( rc != VINF_SUCCESS
5170	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
5171	break;
5172	if (++(*pcLoops) >= pVM->hm.s.cMaxResumeLoops)
5173	{
5174	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
5175	rc = VINF_EM_RAW_INTERRUPT;
5176	break;
5177	}
5178
5179	/** @todo handle single-stepping */
5180	}
5181
5182	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5183	return rc;
5184	}
5185	#endif
5186
5187
5188	/**
5189	* Runs the guest code using AMD-V.
5190	*
5191	* @returns Strict VBox status code.
5192	* @param pVM The cross context VM structure.
5193	* @param pVCpu The cross context virtual CPU structure.
5194	* @param pCtx Pointer to the guest-CPU context.
5195	*/
5196	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5197	{
5198	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5199	HMSVM_ASSERT_PREEMPT_SAFE();
5200	VMMRZCallRing3SetNotification(pVCpu, hmR0SvmCallRing3Callback, pCtx);
5201
5202	uint32_t cLoops = 0;
5203	int rc;
5204	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5205	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
5206	#endif
5207	{
5208	if (!pVCpu->hm.s.fSingleInstruction)
5209	rc = hmR0SvmRunGuestCodeNormal(pVM, pVCpu, pCtx, &cLoops);
5210	else
5211	rc = hmR0SvmRunGuestCodeStep(pVM, pVCpu, pCtx, &cLoops);
5212	}
5213	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5214	else
5215	{
5216	rc = VINF_SVM_VMRUN;
5217	}
5218
5219	/* Re-check the nested-guest condition here as we may be transitioning from the normal
5220	execution loop into the nested-guest, hence this is not placed in the 'else' part above. */
5221	if (rc == VINF_SVM_VMRUN)
5222	{
5223	rc = hmR0SvmRunGuestCodeNested(pVM, pVCpu, pCtx, &cLoops);
5224	if (rc == VINF_SVM_VMEXIT)
5225	rc = VINF_SUCCESS;
5226	}
5227	#endif
5228
5229	/* Fixup error codes. */
5230	if (rc == VERR_EM_INTERPRETER)
5231	rc = VINF_EM_RAW_EMULATE_INSTR;
5232	else if (rc == VINF_EM_RESET)
5233	rc = VINF_EM_TRIPLE_FAULT;
5234
5235	/* Prepare to return to ring-3. This will remove longjmp notifications. */
5236	rc = hmR0SvmExitToRing3(pVM, pVCpu, pCtx, rc);
5237	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
5238	return rc;
5239	}
5240
5241
5242	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5243	/**
5244	* Determines whether an IOIO intercept is active for the nested-guest or not.
5245	*
5246	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
5247	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
5248	*/
5249	static bool hmR0SvmIsIoInterceptActive(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
5250	{
5251	const uint16_t u16Port = pIoExitInfo->n.u16Port;
5252	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
5253	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
5254	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
5255	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
5256	const bool fRep = pIoExitInfo->n.u1Rep;
5257	const bool fStrIo = pIoExitInfo->n.u1Str;
5258
5259	return HMSvmIsIOInterceptActive(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
5260	NULL /* pIoExitInfo */);
5261	}
5262
5263
5264	/**
5265	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
5266	* SVM_EXIT_INVALID).
5267	*
5268	* @returns VBox status code (informational status codes included).
5269	* @param pVCpu The cross context virtual CPU structure.
5270	* @param pCtx Pointer to the guest-CPU context.
5271	* @param pSvmTransient Pointer to the SVM transient structure.
5272	*/
5273	static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5274	{
5275	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
5276	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5277	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5278
5279	/** @todo Figure out why using IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK instead of
5280	* HMSVM_CPUMCTX_EXTRN_ALL breaks nested guests (XP Pro, DSL etc.), see
5281	* also HMSvmNstGstVmExitNotify(). */
5282	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_pCtx, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
5283	do { \
5284	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5285	return VBOXSTRICTRC_TODO(IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2))); \
5286	} while (0)
5287
5288	/*
5289	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
5290	* nested-guest. If it isn't, it should be handled by the (outer) guest.
5291	*/
5292	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
5293	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
5294	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
5295	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
5296	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
5297
5298	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
5299	switch (uExitCode)
5300	{
5301	case SVM_EXIT_CPUID:
5302	{
5303	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_CPUID))
5304	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5305	return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
5306	}
5307
5308	case SVM_EXIT_RDTSC:
5309	{
5310	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_RDTSC))
5311	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5312	return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
5313	}
5314
5315	case SVM_EXIT_RDTSCP:
5316	{
5317	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_RDTSCP))
5318	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5319	return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
5320	}
5321
5322	case SVM_EXIT_MONITOR:
5323	{
5324	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_MONITOR))
5325	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5326	return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
5327	}
5328
5329	case SVM_EXIT_MWAIT:
5330	{
5331	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_MWAIT))
5332	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5333	return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
5334	}
5335
5336	case SVM_EXIT_HLT:
5337	{
5338	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_HLT))
5339	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5340	return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
5341	}
5342
5343	case SVM_EXIT_MSR:
5344	{
5345	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
5346	{
5347	uint32_t const idMsr = pCtx->ecx;
5348	uint16_t offMsrpm;
5349	uint8_t uMsrpmBit;
5350	int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
5351	if (RT_SUCCESS(rc))
5352	{
5353	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
5354	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
5355
5356	uint8_t const pbMsrBitmap = (uint8_t const )pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
5357	pbMsrBitmap += offMsrpm;
5358	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
5359	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
5360
5361	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
5362	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
5363	{
5364	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5365	}
5366	}
5367	else
5368	{
5369	/*
5370	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
5371	* See AMD-V spec. "15.11 MSR Intercepts".
5372	*/
5373	Assert(rc == VERR_OUT_OF_RANGE);
5374	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5375	}
5376	}
5377	return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
5378	}
5379
5380	case SVM_EXIT_IOIO:
5381	{
5382	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
5383	{
5384	void *pvIoBitmap = pCtx->hwvirt.svm.CTX_SUFF(pvIoBitmap);
5385	SVMIOIOEXITINFO IoExitInfo;
5386	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
5387	bool const fIntercept = hmR0SvmIsIoInterceptActive(pvIoBitmap, &IoExitInfo);
5388	if (fIntercept)
5389	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5390	}
5391	return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
5392	}
5393
5394	case SVM_EXIT_XCPT_PF:
5395	{
5396	PVM pVM = pVCpu->CTX_SUFF(pVM);
5397	if (pVM->hm.s.fNestedPaging)
5398	{
5399	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
5400	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
5401
5402	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
5403	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_PF))
5404	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, u32ErrCode, uFaultAddress);
5405
5406	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
5407	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
5408	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
5409	return VINF_SUCCESS;
5410	}
5411	return hmR0SvmExitXcptPF(pVCpu, pCtx,pSvmTransient);
5412	}
5413
5414	case SVM_EXIT_XCPT_UD:
5415	{
5416	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_UD))
5417	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5418	hmR0SvmSetPendingXcptUD(pVCpu);
5419	return VINF_SUCCESS;
5420	}
5421
5422	case SVM_EXIT_XCPT_MF:
5423	{
5424	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_MF))
5425	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5426	return hmR0SvmExitXcptMF(pVCpu, pCtx, pSvmTransient);
5427	}
5428
5429	case SVM_EXIT_XCPT_DB:
5430	{
5431	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_DB))
5432	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5433	return hmR0SvmNestedExitXcptDB(pVCpu, pCtx, pSvmTransient);
5434	}
5435
5436	case SVM_EXIT_XCPT_AC:
5437	{
5438	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_AC))
5439	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5440	return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
5441	}
5442
5443	case SVM_EXIT_XCPT_BP:
5444	{
5445	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_BP))
5446	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5447	return hmR0SvmNestedExitXcptBP(pVCpu, pCtx, pSvmTransient);
5448	}
5449
5450	case SVM_EXIT_READ_CR0:
5451	case SVM_EXIT_READ_CR3:
5452	case SVM_EXIT_READ_CR4:
5453	{
5454	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5455	if (HMIsGuestSvmReadCRxInterceptSet(pVCpu, uCr))
5456	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5457	return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
5458	}
5459
5460	case SVM_EXIT_CR0_SEL_WRITE:
5461	{
5462	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5463	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5464	return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5465	}
5466
5467	case SVM_EXIT_WRITE_CR0:
5468	case SVM_EXIT_WRITE_CR3:
5469	case SVM_EXIT_WRITE_CR4:
5470	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5471	{
5472	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5473	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5474
5475	if (HMIsGuestSvmWriteCRxInterceptSet(pVCpu, uCr))
5476	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5477	return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5478	}
5479
5480	case SVM_EXIT_PAUSE:
5481	{
5482	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_PAUSE))
5483	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5484	return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
5485	}
5486
5487	case SVM_EXIT_VINTR:
5488	{
5489	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VINTR))
5490	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5491	return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5492	}
5493
5494	case SVM_EXIT_INTR:
5495	case SVM_EXIT_NMI:
5496	case SVM_EXIT_SMI:
5497	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5498	{
5499	/*
5500	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5501	*
5502	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5503	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5504	* nested-guest #VMEXIT.
5505	*/
5506	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
5507	return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5508	}
5509
5510	case SVM_EXIT_FERR_FREEZE:
5511	{
5512	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5513	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5514	return hmR0SvmExitFerrFreeze(pVCpu, pCtx, pSvmTransient);
5515	}
5516
5517	case SVM_EXIT_INVLPG:
5518	{
5519	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_INVLPG))
5520	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5521	return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5522	}
5523
5524	case SVM_EXIT_WBINVD:
5525	{
5526	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_WBINVD))
5527	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5528	return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5529	}
5530
5531	case SVM_EXIT_INVD:
5532	{
5533	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_INVD))
5534	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5535	return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5536	}
5537
5538	case SVM_EXIT_RDPMC:
5539	{
5540	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_RDPMC))
5541	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5542	return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5543	}
5544
5545	default:
5546	{
5547	switch (uExitCode)
5548	{
5549	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5550	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5551	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5552	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5553	{
5554	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5555	if (HMIsGuestSvmReadDRxInterceptSet(pVCpu, uDr))
5556	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5557	return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5558	}
5559
5560	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5561	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5562	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5563	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5564	{
5565	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5566	if (HMIsGuestSvmWriteDRxInterceptSet(pVCpu, uDr))
5567	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5568	return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5569	}
5570
5571	case SVM_EXIT_XCPT_DE:
5572	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5573	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5574	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5575	case SVM_EXIT_XCPT_OF:
5576	case SVM_EXIT_XCPT_BR:
5577	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5578	case SVM_EXIT_XCPT_NM:
5579	case SVM_EXIT_XCPT_DF:
5580	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5581	case SVM_EXIT_XCPT_TS:
5582	case SVM_EXIT_XCPT_NP:
5583	case SVM_EXIT_XCPT_SS:
5584	case SVM_EXIT_XCPT_GP:
5585	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5586	case SVM_EXIT_XCPT_15: /* Reserved. */
5587	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5588	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5589	case SVM_EXIT_XCPT_MC:
5590	case SVM_EXIT_XCPT_XF:
5591	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5592	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5593	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5594	{
5595	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5596	if (HMIsGuestSvmXcptInterceptSet(pVCpu, uVector))
5597	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5598	return hmR0SvmExitXcptGeneric(pVCpu, pCtx, pSvmTransient);
5599	}
5600
5601	case SVM_EXIT_XSETBV:
5602	{
5603	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_XSETBV))
5604	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5605	return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5606	}
5607
5608	case SVM_EXIT_TASK_SWITCH:
5609	{
5610	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5611	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5612	return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5613	}
5614
5615	case SVM_EXIT_IRET:
5616	{
5617	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_IRET))
5618	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5619	return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5620	}
5621
5622	case SVM_EXIT_SHUTDOWN:
5623	{
5624	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
5625	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5626	return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5627	}
5628
5629	case SVM_EXIT_VMMCALL:
5630	{
5631	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VMMCALL))
5632	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5633	return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5634	}
5635
5636	case SVM_EXIT_CLGI:
5637	{
5638	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_CLGI))
5639	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5640	return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5641	}
5642
5643	case SVM_EXIT_STGI:
5644	{
5645	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_STGI))
5646	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5647	return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5648	}
5649
5650	case SVM_EXIT_VMLOAD:
5651	{
5652	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VMLOAD))
5653	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5654	return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5655	}
5656
5657	case SVM_EXIT_VMSAVE:
5658	{
5659	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VMSAVE))
5660	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5661	return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5662	}
5663
5664	case SVM_EXIT_INVLPGA:
5665	{
5666	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_INVLPGA))
5667	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5668	return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5669	}
5670
5671	case SVM_EXIT_VMRUN:
5672	{
5673	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VMRUN))
5674	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5675	return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5676	}
5677
5678	case SVM_EXIT_RSM:
5679	{
5680	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_RSM))
5681	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5682	hmR0SvmSetPendingXcptUD(pVCpu);
5683	return VINF_SUCCESS;
5684	}
5685
5686	case SVM_EXIT_SKINIT:
5687	{
5688	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_SKINIT))
5689	NST_GST_VMEXIT_CALL_RET(pVCpu, pCtx, uExitCode, uExitInfo1, uExitInfo2);
5690	hmR0SvmSetPendingXcptUD(pVCpu);
5691	return VINF_SUCCESS;
5692	}
5693
5694	case SVM_EXIT_NPF:
5695	{
5696	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
5697	return hmR0SvmExitNestedPF(pVCpu, pCtx, pSvmTransient);
5698	}
5699
5700	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5701	return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5702
5703	default:
5704	{
5705	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5706	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5707	return VERR_SVM_UNKNOWN_EXIT;
5708	}
5709	}
5710	}
5711	}
5712	/* not reached */
5713
5714	#undef NST_GST_VMEXIT_CALL_RET
5715	}
5716	#endif
5717
5718
5719	/**
5720	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5721	*
5722	* @returns VBox status code (informational status codes included).
5723	* @param pVCpu The cross context virtual CPU structure.
5724	* @param pCtx Pointer to the guest-CPU context.
5725	* @param pSvmTransient Pointer to the SVM transient structure.
5726	*/
5727	static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5728	{
5729	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5730	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5731
5732	#ifdef DEBUG_ramshankar
5733	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5734	do { \
5735	if ((a_fDbg) == 1) \
5736	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5737	int rc = a_CallExpr; \
5738	if ((a_fDbg) == 1) \
5739	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5740	return rc; \
5741	} while (0)
5742	#else
5743	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5744	#endif
5745
5746	/*
5747	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5748	* for most guests under normal workloads (for some definition of "normal").
5749	*/
5750	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5751	switch (uExitCode)
5752	{
5753	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pCtx, pSvmTransient));
5754	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient));
5755	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient));
5756	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient));
5757	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient));
5758	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pCtx, pSvmTransient));
5759	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient));
5760	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient));
5761	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient));
5762	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient));
5763
5764	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5765	case SVM_EXIT_INTR:
5766	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient));
5767
5768	case SVM_EXIT_READ_CR0:
5769	case SVM_EXIT_READ_CR3:
5770	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient));
5771
5772	case SVM_EXIT_CR0_SEL_WRITE:
5773	case SVM_EXIT_WRITE_CR0:
5774	case SVM_EXIT_WRITE_CR3:
5775	case SVM_EXIT_WRITE_CR4:
5776	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient));
5777
5778	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pCtx, pSvmTransient));
5779	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient));
5780	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient));
5781	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient));
5782	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient));
5783	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient));
5784	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient));
5785	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient));
5786	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pCtx, pSvmTransient));
5787	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pCtx, pSvmTransient));
5788	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pCtx, pSvmTransient));
5789	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient));
5790	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pCtx, pSvmTransient));
5791	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient));
5792	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pCtx, pSvmTransient));
5793
5794	default:
5795	{
5796	switch (pSvmTransient->u64ExitCode)
5797	{
5798	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5799	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5800	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5801	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5802	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient));
5803
5804	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5805	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5806	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5807	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5808	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient));
5809
5810	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient));
5811	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient));
5812
5813	case SVM_EXIT_SMI:
5814	case SVM_EXIT_INIT:
5815	{
5816	/*
5817	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5818	* If it ever does, we want to know about it so log the exit code and bail.
5819	*/
5820	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient));
5821	}
5822
5823	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5824	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient));
5825	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient));
5826	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient));
5827	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient));
5828	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient));
5829	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient));
5830	#else
5831	case SVM_EXIT_CLGI:
5832	case SVM_EXIT_STGI:
5833	case SVM_EXIT_VMLOAD:
5834	case SVM_EXIT_VMSAVE:
5835	case SVM_EXIT_INVLPGA:
5836	case SVM_EXIT_VMRUN:
5837	#endif
5838	case SVM_EXIT_RSM:
5839	case SVM_EXIT_SKINIT:
5840	{
5841	hmR0SvmSetPendingXcptUD(pVCpu);
5842	return VINF_SUCCESS;
5843	}
5844
5845	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
5846	case SVM_EXIT_XCPT_DE:
5847	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5848	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5849	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5850	case SVM_EXIT_XCPT_OF:
5851	case SVM_EXIT_XCPT_BR:
5852	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5853	case SVM_EXIT_XCPT_NM:
5854	case SVM_EXIT_XCPT_DF:
5855	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5856	case SVM_EXIT_XCPT_TS:
5857	case SVM_EXIT_XCPT_NP:
5858	case SVM_EXIT_XCPT_SS:
5859	case SVM_EXIT_XCPT_GP:
5860	/* SVM_EXIT_XCPT_PF: */
5861	case SVM_EXIT_XCPT_15: /* Reserved. */
5862	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5863	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5864	case SVM_EXIT_XCPT_MC:
5865	case SVM_EXIT_XCPT_XF:
5866	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5867	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5868	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5869	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pCtx, pSvmTransient));
5870	#endif /* HMSVM_ALWAYS_TRAP_ALL_XCPTS */
5871
5872	default:
5873	{
5874	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5875	pVCpu->hm.s.u32HMError = uExitCode;
5876	return VERR_SVM_UNKNOWN_EXIT;
5877	}
5878	}
5879	}
5880	}
5881	/* not reached */
5882	#undef VMEXIT_CALL_RET
5883	}
5884
5885
5886	#ifdef DEBUG
5887	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5888	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
5889	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5890
5891	# define HMSVM_ASSERT_PREEMPT_CPUID() \
5892	do \
5893	{ \
5894	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5895	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5896	} while (0)
5897
5898	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() \
5899	do { \
5900	AssertPtr(pVCpu); \
5901	AssertPtr(pCtx); \
5902	AssertPtr(pSvmTransient); \
5903	Assert(ASMIntAreEnabled()); \
5904	HMSVM_ASSERT_PREEMPT_SAFE(); \
5905	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
5906	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (uint32_t)pVCpu->idCpu)); \
5907	HMSVM_ASSERT_PREEMPT_SAFE(); \
5908	if (VMMR0IsLogFlushDisabled(pVCpu)) \
5909	HMSVM_ASSERT_PREEMPT_CPUID(); \
5910	} while (0)
5911	#else /* Release builds */
5912	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() do { NOREF(pVCpu); NOREF(pCtx); NOREF(pSvmTransient); } while (0)
5913	#endif
5914
5915
5916	/**
5917	* Worker for hmR0SvmInterpretInvlpg().
5918	*
5919	* @return VBox status code.
5920	* @param pVCpu The cross context virtual CPU structure.
5921	* @param pCpu Pointer to the disassembler state.
5922	* @param pCtx The guest CPU context.
5923	*/
5924	static int hmR0SvmInterpretInvlPgEx(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTX pCtx)
5925	{
5926	DISQPVPARAMVAL Param1;
5927	RTGCPTR GCPtrPage;
5928
5929	int rc = DISQueryParamVal(CPUMCTX2CORE(pCtx), pCpu, &pCpu->Param1, &Param1, DISQPVWHICH_SRC);
5930	if (RT_FAILURE(rc))
5931	return VERR_EM_INTERPRETER;
5932
5933	if ( Param1.type == DISQPV_TYPE_IMMEDIATE
5934	\|\| Param1.type == DISQPV_TYPE_ADDRESS)
5935	{
5936	if (!(Param1.flags & (DISQPV_FLAG_32 \| DISQPV_FLAG_64)))
5937	return VERR_EM_INTERPRETER;
5938
5939	GCPtrPage = Param1.val.val64;
5940	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), GCPtrPage);
5941	rc = VBOXSTRICTRC_VAL(rc2);
5942	}
5943	else
5944	{
5945	Log4Func(("Invalid parameter type %#x\n", Param1.type));
5946	rc = VERR_EM_INTERPRETER;
5947	}
5948
5949	return rc;
5950	}
5951
5952
5953	/**
5954	* Interprets INVLPG.
5955	*
5956	* @returns VBox status code.
5957	* @retval VINF_* Scheduling instructions.
5958	* @retval VERR_EM_INTERPRETER Something we can't cope with.
5959	* @retval VERR_* Fatal errors.
5960	*
5961	* @param pVM The cross context VM structure.
5962	* @param pVCpu The cross context virtual CPU structure.
5963	* @param pCtx The guest CPU context.
5964	*
5965	* @remarks Updates the RIP if the instruction was executed successfully.
5966	*/
5967	static int hmR0SvmInterpretInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5968	{
5969	/* Only allow 32 & 64 bit code. */
5970	if (CPUMGetGuestCodeBits(pVCpu) != 16)
5971	{
5972	PDISSTATE pDis = &pVCpu->hm.s.DisState;
5973	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
5974	if ( RT_SUCCESS(rc)
5975	&& pDis->pCurInstr->uOpcode == OP_INVLPG)
5976	{
5977	rc = hmR0SvmInterpretInvlPgEx(pVCpu, pDis, pCtx);
5978	if (RT_SUCCESS(rc))
5979	pCtx->rip += pDis->cbInstr;
5980	return rc;
5981	}
5982	else
5983	Log4Func(("EMInterpretDisasCurrent failed! rc=%Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
5984	}
5985	return VERR_EM_INTERPRETER;
5986	}
5987
5988
5989	/**
5990	* Gets the IEM exception flags for the specified SVM event.
5991	*
5992	* @returns The IEM exception flags.
5993	* @param pEvent Pointer to the SVM event.
5994	*
5995	* @remarks This function currently only constructs flags required for
5996	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
5997	* and CR2 aspects of an exception are not included).
5998	*/
5999	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6000	{
6001	uint8_t const uEventType = pEvent->n.u3Type;
6002	uint32_t fIemXcptFlags;
6003	switch (uEventType)
6004	{
6005	case SVM_EVENT_EXCEPTION:
6006	/*
6007	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6008	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6009	*/
6010	if (pEvent->n.u8Vector == X86_XCPT_BP)
6011	{
6012	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6013	break;
6014	}
6015	if (pEvent->n.u8Vector == X86_XCPT_OF)
6016	{
6017	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6018	break;
6019	}
6020	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6021	RT_FALL_THRU();
6022	case SVM_EVENT_NMI:
6023	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6024	break;
6025
6026	case SVM_EVENT_EXTERNAL_IRQ:
6027	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6028	break;
6029
6030	case SVM_EVENT_SOFTWARE_INT:
6031	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6032	break;
6033
6034	default:
6035	fIemXcptFlags = 0;
6036	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6037	break;
6038	}
6039	return fIemXcptFlags;
6040	}
6041
6042
6043	/**
6044	* Handle a condition that occurred while delivering an event through the guest
6045	* IDT.
6046	*
6047	* @returns VBox status code (informational error codes included).
6048	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6049	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6050	* continue execution of the guest which will delivery the \#DF.
6051	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6052	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6053	*
6054	* @param pVCpu The cross context virtual CPU structure.
6055	* @param pCtx Pointer to the guest-CPU context.
6056	* @param pSvmTransient Pointer to the SVM transient structure.
6057	*
6058	* @remarks No-long-jump zone!!!
6059	*/
6060	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6061	{
6062	int rc = VINF_SUCCESS;
6063	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6064	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6065
6066	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6067	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6068	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6069
6070	/*
6071	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6072	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6073	*
6074	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6075	*/
6076	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6077	{
6078	IEMXCPTRAISE enmRaise;
6079	IEMXCPTRAISEINFO fRaiseInfo;
6080	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6081	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6082	if (fExitIsHwXcpt)
6083	{
6084	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6085	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6086	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6087	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6088	}
6089	else
6090	{
6091	/*
6092	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6093	* then we end up here.
6094	*
6095	* If the event was:
6096	* - a software interrupt, we can re-execute the instruction which will
6097	* regenerate the event.
6098	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6099	* - a hardware exception or external interrupt, we re-inject it.
6100	*/
6101	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6102	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6103	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6104	else
6105	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6106	}
6107
6108	switch (enmRaise)
6109	{
6110	case IEMXCPTRAISE_CURRENT_XCPT:
6111	case IEMXCPTRAISE_PREV_EVENT:
6112	{
6113	/* For software interrupts, we shall re-execute the instruction. */
6114	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6115	{
6116	RTGCUINTPTR GCPtrFaultAddress = 0;
6117
6118	/* If we are re-injecting an NMI, clear NMI blocking. */
6119	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6120	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6121
6122	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6123	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6124	{
6125	pSvmTransient->fVectoringPF = true;
6126	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n", pCtx->cr2));
6127	}
6128	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6129	&& uIdtVector == X86_XCPT_PF)
6130	{
6131	/*
6132	* If the previous exception was a #PF, we need to recover the CR2 value.
6133	* This can't happen with shadow paging.
6134	*/
6135	GCPtrFaultAddress = pCtx->cr2;
6136	}
6137
6138	/*
6139	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6140	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6141	*/
6142	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6143	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6144	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6145
6146	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6147	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6148	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6149	}
6150	break;
6151	}
6152
6153	case IEMXCPTRAISE_REEXEC_INSTR:
6154	{
6155	Assert(rc == VINF_SUCCESS);
6156	break;
6157	}
6158
6159	case IEMXCPTRAISE_DOUBLE_FAULT:
6160	{
6161	/*
6162	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6163	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6164	* converted into a #DF.
6165	*/
6166	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6167	{
6168	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pCtx->cr2));
6169	pSvmTransient->fVectoringDoublePF = true;
6170	Assert(rc == VINF_SUCCESS);
6171	}
6172	else
6173	{
6174	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6175	hmR0SvmSetPendingXcptDF(pVCpu);
6176	rc = VINF_HM_DOUBLE_FAULT;
6177	}
6178	break;
6179	}
6180
6181	case IEMXCPTRAISE_TRIPLE_FAULT:
6182	{
6183	rc = VINF_EM_RESET;
6184	break;
6185	}
6186
6187	case IEMXCPTRAISE_CPU_HANG:
6188	{
6189	rc = VERR_EM_GUEST_CPU_HANG;
6190	break;
6191	}
6192
6193	default:
6194	{
6195	AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6196	rc = VERR_SVM_IPE_2;
6197	break;
6198	}
6199	}
6200	}
6201	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6202	NOREF(pCtx);
6203	return rc;
6204	}
6205
6206
6207	/**
6208	* Advances the guest RIP making use of the CPU's NRIP_SAVE feature if
6209	* supported, otherwise advances the RIP by the number of bytes specified in
6210	* @a cb.
6211	*
6212	* @param pVCpu The cross context virtual CPU structure.
6213	* @param pCtx Pointer to the guest-CPU context.
6214	* @param cb RIP increment value in bytes.
6215	*
6216	* @remarks Use this function only from \#VMEXIT's where the NRIP value is valid
6217	* when NRIP_SAVE is supported by the CPU, otherwise use
6218	* hmR0SvmAdvanceRipDumb!
6219	*/
6220	DECLINLINE(void) hmR0SvmAdvanceRipHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
6221	{
6222	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6223	if (fSupportsNextRipSave)
6224	{
6225	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6226	Assert(pVmcb);
6227	Assert(pVmcb->ctrl.u64NextRIP);
6228	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
6229	AssertRelease(pVmcb->ctrl.u64NextRIP - pCtx->rip == cb); /* temporary, remove later */
6230	pCtx->rip = pVmcb->ctrl.u64NextRIP;
6231	}
6232	else
6233	pCtx->rip += cb;
6234
6235	HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
6236	}
6237
6238
6239	/**
6240	* Gets the length of the current instruction if the CPU supports the NRIP_SAVE
6241	* feature. Otherwise, returns the value in @a cbLikely.
6242	*
6243	* @param pVCpu The cross context virtual CPU structure.
6244	* @param pCtx Pointer to the guest-CPU context.
6245	* @param cbLikely The likely instruction length.
6246	*/
6247	DECLINLINE(uint8_t) hmR0SvmGetInstrLengthHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint8_t cbLikely)
6248	{
6249	Assert(cbLikely <= 15); /* See Intel spec. 2.3.11 "AVX Instruction Length" */
6250	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6251	if (fSupportsNextRipSave)
6252	{
6253	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6254	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6255	Assert(cbInstr == cbLikely);
6256	return cbInstr;
6257	}
6258	return cbLikely;
6259	}
6260
6261
6262	/**
6263	* Advances the guest RIP by the number of bytes specified in @a cb. This does
6264	* not make use of any hardware features to determine the instruction length.
6265	*
6266	* @param pVCpu The cross context virtual CPU structure.
6267	* @param pCtx Pointer to the guest-CPU context.
6268	* @param cb RIP increment value in bytes.
6269	*/
6270	DECLINLINE(void) hmR0SvmAdvanceRipDumb(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
6271	{
6272	pCtx->rip += cb;
6273	HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
6274	}
6275	#undef HMSVM_UPDATE_INTR_SHADOW
6276
6277
6278	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6279	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6280	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6281
6282	/** @name \#VMEXIT handlers.
6283	* @{
6284	*/
6285
6286	/**
6287	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6288	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6289	*/
6290	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6291	{
6292	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6293
6294	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6295	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6296	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6297	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6298
6299	/*
6300	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6301	* signal -before- the timer fires if the current interrupt is our own timer or a some
6302	* other host interrupt. We also cannot examine what interrupt it is until the host
6303	* actually take the interrupt.
6304	*
6305	* Going back to executing guest code here unconditionally causes random scheduling
6306	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6307	*/
6308	return VINF_EM_RAW_INTERRUPT;
6309	}
6310
6311
6312	/**
6313	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6314	*/
6315	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6316	{
6317	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6318
6319	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6320	int rc = VINF_SUCCESS;
6321	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6322	return rc;
6323	}
6324
6325
6326	/**
6327	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6328	*/
6329	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6330	{
6331	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6332
6333	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6334	int rc = VINF_SUCCESS;
6335	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6336	return rc;
6337	}
6338
6339
6340	/**
6341	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6342	*/
6343	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6344	{
6345	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6346
6347	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP
6348	\| CPUMCTX_EXTRN_CS);
6349	VBOXSTRICTRC rcStrict;
6350	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6351	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6352	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6353	if (!pExitRec)
6354	{
6355	PVM pVM = pVCpu->CTX_SUFF(pVM);
6356	rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6357	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
6358	{
6359	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6360	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6361	}
6362	else
6363	{
6364	AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6365	rcStrict = VERR_EM_INTERPRETER;
6366	}
6367	}
6368	else
6369	{
6370	/*
6371	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6372	*/
6373	Assert(pCtx == &pVCpu->cpum.GstCtx);
6374	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6375
6376	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6377	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6378
6379	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6380
6381	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6382	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6383	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6384	}
6385	return VBOXSTRICTRC_TODO(rcStrict);
6386	}
6387
6388
6389	/**
6390	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6391	*/
6392	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6393	{
6394	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6395	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6396	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 2));
6397	if (rcStrict == VINF_SUCCESS)
6398	pSvmTransient->fUpdateTscOffsetting = true;
6399	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6400	rcStrict = VINF_SUCCESS;
6401	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6402	return VBOXSTRICTRC_TODO(rcStrict);
6403	}
6404
6405
6406	/**
6407	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6408	*/
6409	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6410	{
6411	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6412	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3));
6413	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6414	if (rcStrict == VINF_SUCCESS)
6415	pSvmTransient->fUpdateTscOffsetting = true;
6416	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6417	rcStrict = VINF_SUCCESS;
6418	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6419	return VBOXSTRICTRC_TODO(rcStrict);
6420	}
6421
6422
6423	/**
6424	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6425	*/
6426	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6427	{
6428	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6429	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0
6430	\| CPUMCTX_EXTRN_CR4
6431	\| CPUMCTX_EXTRN_SS);
6432
6433	int rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6434	if (RT_LIKELY(rc == VINF_SUCCESS))
6435	{
6436	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6437	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6438	}
6439	else
6440	{
6441	AssertMsgFailed(("hmR0SvmExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6442	rc = VERR_EM_INTERPRETER;
6443	}
6444	return rc;
6445	}
6446
6447
6448	/**
6449	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
6450	*/
6451	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6452	{
6453	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6454	PVM pVM = pVCpu->CTX_SUFF(pVM);
6455	Assert(!pVM->hm.s.fNestedPaging);
6456
6457	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6458	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6459	if ( fSupportsDecodeAssists
6460	&& fSupportsNextRipSave)
6461	{
6462	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6463	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6464	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6465	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
6466	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
6467	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6468	return VBOXSTRICTRC_VAL(rcStrict);
6469	}
6470
6471	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6472	int rc = hmR0SvmInterpretInvlpg(pVM, pVCpu, pCtx); /* Updates RIP if successful. */
6473	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
6474	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6475	return rc;
6476	}
6477
6478
6479	/**
6480	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
6481	*/
6482	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6483	{
6484	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6485
6486	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 1);
6487	int rc = EMShouldContinueAfterHalt(pVCpu, pCtx) ? VINF_SUCCESS : VINF_EM_HALT;
6488	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6489
6490	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
6491	if (rc != VINF_SUCCESS)
6492	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
6493	return rc;
6494	}
6495
6496
6497	/**
6498	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
6499	*/
6500	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6501	{
6502	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6503	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0
6504	\| CPUMCTX_EXTRN_SS);
6505
6506	int rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6507	if (RT_LIKELY(rc == VINF_SUCCESS))
6508	{
6509	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6510	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6511	}
6512	else
6513	{
6514	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
6515	rc = VERR_EM_INTERPRETER;
6516	}
6517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6518	return rc;
6519	}
6520
6521
6522	/**
6523	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
6524	*/
6525	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6526	{
6527	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6528	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0
6529	\| CPUMCTX_EXTRN_SS);
6530
6531	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6532	int rc = VBOXSTRICTRC_VAL(rc2);
6533	if ( rc == VINF_EM_HALT
6534	\|\| rc == VINF_SUCCESS)
6535	{
6536	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6537
6538	if ( rc == VINF_EM_HALT
6539	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
6540	{
6541	rc = VINF_SUCCESS;
6542	}
6543	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6544	}
6545	else
6546	{
6547	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
6548	rc = VERR_EM_INTERPRETER;
6549	}
6550	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
6551	("hmR0SvmExitMwait: EMInterpretMWait failed rc=%Rrc\n", rc));
6552	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
6553	return rc;
6554	}
6555
6556
6557	/**
6558	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
6559	* \#VMEXIT.
6560	*/
6561	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6562	{
6563	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6564	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6565	return VINF_EM_RESET;
6566	}
6567
6568
6569	/**
6570	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
6571	*/
6572	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6573	{
6574	RT_NOREF(pCtx);
6575	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6576	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6577	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
6578	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
6579	RT_NOREF(pVmcb);
6580	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6581	return VERR_SVM_UNEXPECTED_EXIT;
6582	}
6583
6584
6585	/**
6586	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
6587	*/
6588	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6589	{
6590	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6591
6592	Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6593	#ifdef VBOX_WITH_STATISTICS
6594	switch (pSvmTransient->u64ExitCode)
6595	{
6596	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
6597	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
6598	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
6599	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
6600	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
6601	}
6602	#endif
6603
6604	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6605	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6606	if ( fSupportsDecodeAssists
6607	&& fSupportsNextRipSave)
6608	{
6609	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6610	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6611	if (fMovCRx)
6612	{
6613	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6614	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6615	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
6616	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6617	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6618	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6619	return VBOXSTRICTRC_VAL(rcStrict);
6620	}
6621	/* else: SMSW instruction, fall back below to IEM for this. */
6622	}
6623
6624	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6625	VBOXSTRICTRC rc2 = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6626	int rc = VBOXSTRICTRC_VAL(rc2);
6627	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3,
6628	("hmR0SvmExitReadCRx: EMInterpretInstruction failed rc=%Rrc\n", rc));
6629	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
6630	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6631	return rc;
6632	}
6633
6634
6635	/**
6636	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
6637	*/
6638	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6639	{
6640	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6641
6642	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6643	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
6644	Assert(iCrReg <= 15);
6645
6646	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
6647	bool fDecodedInstr = false;
6648	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6649	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6650	if ( fSupportsDecodeAssists
6651	&& fSupportsNextRipSave)
6652	{
6653	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6654	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6655	if (fMovCRx)
6656	{
6657	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6658	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6659	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6660	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
6661	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6662	fDecodedInstr = true;
6663	}
6664	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
6665	}
6666
6667	if (!fDecodedInstr)
6668	{
6669	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6670	Log4Func(("iCrReg=%#x\n", iCrReg));
6671	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(pCtx), NULL);
6672	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
6673	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
6674	rcStrict = VERR_EM_INTERPRETER;
6675	}
6676
6677	if (rcStrict == VINF_SUCCESS)
6678	{
6679	switch (iCrReg)
6680	{
6681	case 0:
6682	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
6683	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
6684	break;
6685
6686	case 2:
6687	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
6688	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
6689	break;
6690
6691	case 3:
6692	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
6693	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
6694	break;
6695
6696	case 4:
6697	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
6698	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
6699	break;
6700
6701	case 8:
6702	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
6704	break;
6705
6706	default:
6707	{
6708	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
6709	pSvmTransient->u64ExitCode, iCrReg));
6710	break;
6711	}
6712	}
6713	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6714	}
6715	else
6716	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_CHANGE_MODE \|\| rcStrict == VINF_PGM_SYNC_CR3);
6717	return VBOXSTRICTRC_TODO(rcStrict);
6718	}
6719
6720
6721	/**
6722	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
6723	*
6724	* @returns VBox status code.
6725	* @param pVCpu The cross context virtual CPU structure.
6726	* @param pVmcb Pointer to the VM control block.
6727	*/
6728	static int hmR0SvmExitReadMsr(PVMCPU pVCpu, PSVMVMCB pVmcb)
6729	{
6730	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6731	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0
6732	\| CPUMCTX_EXTRN_RFLAGS
6733	\| CPUMCTX_EXTRN_SS
6734	\| CPUMCTX_EXTRN_ALL_MSRS);
6735
6736	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
6737	Log4Func(("idMsr=%#RX32\n", pCtx->ecx));
6738
6739	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6740	if (fSupportsNextRipSave)
6741	{
6742	int rc = EMInterpretRdmsr(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6743	if (RT_LIKELY(rc == VINF_SUCCESS))
6744	{
6745	pCtx->rip = pVmcb->ctrl.u64NextRIP;
6746	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6747	return VINF_SUCCESS;
6748	}
6749
6750	AssertMsg( rc == VERR_EM_INTERPRETER
6751	\|\| rc == VINF_CPUM_R3_MSR_READ, ("EMInterpretRdmsr failed rc=%Rrc\n", rc));
6752	return rc;
6753	}
6754
6755	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6756	int rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0));
6757	if (RT_UNLIKELY(rc != VINF_SUCCESS))
6758	{
6759	AssertMsg( rc == VERR_EM_INTERPRETER
6760	\|\| rc == VINF_CPUM_R3_MSR_READ, ("EMInterpretInstruction failed rc=%Rrc\n", rc));
6761	}
6762	/* RIP updated by EMInterpretInstruction(). */
6763	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6764	return rc;
6765	}
6766
6767
6768	/**
6769	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
6770	*
6771	* @returns VBox status code.
6772	* @param pVCpu The cross context virtual CPU structure.
6773	* @param pVmcb Pointer to the VM control block.
6774	* @param pSvmTransient Pointer to the SVM-transient structure.
6775	*/
6776	static int hmR0SvmExitWriteMsr(PVMCPU pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
6777	{
6778	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6779	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0
6780	\| CPUMCTX_EXTRN_RFLAGS
6781	\| CPUMCTX_EXTRN_SS
6782	\| CPUMCTX_EXTRN_ALL_MSRS);
6783
6784	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
6785	Log4Func(("idMsr=%#RX32\n", pCtx->ecx));
6786
6787	/*
6788	* Handle TPR patching MSR writes.
6789	* We utilitize the LSTAR MSR for patching.
6790	*/
6791	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive
6792	&& pCtx->ecx == MSR_K8_LSTAR)
6793	{
6794	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
6795	{
6796	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
6797	int rc2 = APICSetTpr(pVCpu, pCtx->eax & 0xff);
6798	AssertRC(rc2);
6799	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6800	}
6801
6802	int rc = VINF_SUCCESS;
6803	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6804	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6805	return rc;
6806	}
6807
6808	/*
6809	* Handle regular MSR writes.
6810	*/
6811	int rc;
6812	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6813	if (fSupportsNextRipSave)
6814	{
6815	rc = EMInterpretWrmsr(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6816	if (RT_LIKELY(rc == VINF_SUCCESS))
6817	{
6818	pCtx->rip = pVmcb->ctrl.u64NextRIP;
6819	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6820	}
6821	else
6822	AssertMsg( rc == VERR_EM_INTERPRETER
6823	\|\| rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: EMInterpretWrmsr failed rc=%Rrc\n", rc));
6824	}
6825	else
6826	{
6827	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6828	rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */));
6829	if (RT_LIKELY(rc == VINF_SUCCESS))
6830	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc); /* RIP updated by EMInterpretInstruction(). */
6831	else
6832	AssertMsg( rc == VERR_EM_INTERPRETER
6833	\|\| rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: WrMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6834	}
6835
6836	if (rc == VINF_SUCCESS)
6837	{
6838	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
6839	if ( pCtx->ecx >= MSR_IA32_X2APIC_START
6840	&& pCtx->ecx <= MSR_IA32_X2APIC_END)
6841	{
6842	/*
6843	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
6844	* When full APIC register virtualization is implemented we'll have to make sure
6845	* APIC state is saved from the VMCB before EMInterpretWrmsr() changes it.
6846	*/
6847	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6848	}
6849	else
6850	{
6851	switch (pCtx->ecx)
6852	{
6853	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
6854	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
6855	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
6856	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
6857	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
6858	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
6859	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
6860	}
6861	}
6862	}
6863
6864	/* RIP has been updated by above after EMInterpretWrmsr() or by EMInterpretInstruction(). */
6865	return rc;
6866	}
6867
6868
6869	/**
6870	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
6871	* \#VMEXIT.
6872	*/
6873	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6874	{
6875	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6876
6877	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6878	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
6879	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
6880
6881	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
6882	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
6883	}
6884
6885
6886	/**
6887	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
6888	*/
6889	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6890	{
6891	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6892	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6893
6894	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
6895
6896	/** @todo Stepping with nested-guest. */
6897	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
6898	{
6899	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
6900	if (pSvmTransient->fWasGuestDebugStateActive)
6901	{
6902	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
6903	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6904	return VERR_SVM_UNEXPECTED_EXIT;
6905	}
6906
6907	/*
6908	* Lazy DR0-3 loading.
6909	*/
6910	if (!pSvmTransient->fWasHyperDebugStateActive)
6911	{
6912	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
6913	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
6914
6915	/* Don't intercept DRx read and writes. */
6916	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6917	pVmcb->ctrl.u16InterceptRdDRx = 0;
6918	pVmcb->ctrl.u16InterceptWrDRx = 0;
6919	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
6920
6921	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6922	VMMRZCallRing3Disable(pVCpu);
6923	HM_DISABLE_PREEMPT();
6924
6925	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
6926	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
6927	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
6928
6929	HM_RESTORE_PREEMPT();
6930	VMMRZCallRing3Enable(pVCpu);
6931
6932	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
6933	return VINF_SUCCESS;
6934	}
6935	}
6936
6937	/*
6938	* Interpret the read/writing of DRx.
6939	*/
6940	/** @todo Decode assist. */
6941	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6942	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
6943	if (RT_LIKELY(rc == VINF_SUCCESS))
6944	{
6945	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
6946	/** @todo CPUM should set this flag! */
6947	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
6948	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6949	}
6950	else
6951	Assert(rc == VERR_EM_INTERPRETER);
6952	return VBOXSTRICTRC_TODO(rc);
6953	}
6954
6955
6956	/**
6957	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
6958	*/
6959	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6960	{
6961	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6962	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
6963	int rc = hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
6964	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
6965	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
6966	return rc;
6967	}
6968
6969
6970	/**
6971	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
6972	*/
6973	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6974	{
6975	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6976	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6977
6978	/** @todo decode assists... */
6979	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6980	if (rcStrict == VINF_IEM_RAISED_XCPT)
6981	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
6982
6983	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6984	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%d (cr4=%RX64) rcStrict=%Rrc\n", pCtx->aXcr[0], pVCpu->hm.s.fLoadSaveGuestXcr0,
6985	pCtx->cr4, VBOXSTRICTRC_VAL(rcStrict)));
6986
6987	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6988	return VBOXSTRICTRC_TODO(rcStrict);
6989	}
6990
6991
6992	/**
6993	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
6994	*/
6995	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6996	{
6997	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6998	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
6999	\| CPUMCTX_EXTRN_SREG_MASK);
7000
7001	/* I/O operation lookup arrays. */
7002	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7003	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7004	the result (in AL/AX/EAX). */
7005	Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
7006
7007	PVM pVM = pVCpu->CTX_SUFF(pVM);
7008	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7009
7010	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7011	SVMIOIOEXITINFO IoExitInfo;
7012	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7013	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7014	uint32_t cbValue = s_aIOSize[uIOWidth];
7015	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7016
7017	if (RT_UNLIKELY(!cbValue))
7018	{
7019	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7020	return VERR_EM_INTERPRETER;
7021	}
7022
7023	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP
7024	\| CPUMCTX_EXTRN_CS
7025	\| CPUMCTX_EXTRN_RFLAGS);
7026	VBOXSTRICTRC rcStrict;
7027	PCEMEXITREC pExitRec = NULL;
7028	if ( !pVCpu->hm.s.fSingleInstruction
7029	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7030	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7031	!IoExitInfo.n.u1Str
7032	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7033	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7034	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7035	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7036	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7037	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7038	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7039	if (!pExitRec)
7040	{
7041	bool fUpdateRipAlready = false;
7042	if (IoExitInfo.n.u1Str)
7043	{
7044	/* INS/OUTS - I/O String instruction. */
7045	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7046	* in EXITINFO1? Investigate once this thing is up and running. */
7047	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7048	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7049	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7050	static IEMMODE const s_aenmAddrMode[8] =
7051	{
7052	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7053	};
7054	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7055	if (enmAddrMode != (IEMMODE)-1)
7056	{
7057	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7058	if (cbInstr <= 15 && cbInstr >= 1)
7059	{
7060	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7061	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7062	{
7063	/* Don't know exactly how to detect whether u3Seg is valid, currently
7064	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7065	2384 Opterons when only checking NRIP. */
7066	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
7067	if ( fSupportsNextRipSave
7068	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7069	{
7070	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7071	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7072	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7073	IoExitInfo.n.u3Seg, true /fIoChecked/);
7074	}
7075	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7076	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7077	X86_SREG_DS, true /fIoChecked/);
7078	else
7079	rcStrict = IEMExecOne(pVCpu);
7080	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7081	}
7082	else
7083	{
7084	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7085	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7086	true /fIoChecked/);
7087	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7088	}
7089	}
7090	else
7091	{
7092	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7093	rcStrict = IEMExecOne(pVCpu);
7094	}
7095	}
7096	else
7097	{
7098	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7099	rcStrict = IEMExecOne(pVCpu);
7100	}
7101	fUpdateRipAlready = true;
7102	}
7103	else
7104	{
7105	/* IN/OUT - I/O instruction. */
7106	Assert(!IoExitInfo.n.u1Rep);
7107
7108	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7109	{
7110	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7112	}
7113	else
7114	{
7115	uint32_t u32Val = 0;
7116	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7117	if (IOM_SUCCESS(rcStrict))
7118	{
7119	/* Save result of I/O IN instr. in AL/AX/EAX. */
7120	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7121	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7122	}
7123	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
7124	HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pVmcb->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port, uAndVal, cbValue);
7125
7126	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7127	}
7128	}
7129
7130	if (IOM_SUCCESS(rcStrict))
7131	{
7132	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7133	if (!fUpdateRipAlready)
7134	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7135
7136	/*
7137	* If any I/O breakpoints are armed, we need to check if one triggered
7138	* and take appropriate action.
7139	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7140	*/
7141	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7142	* execution engines about whether hyper BPs and such are pending. */
7143	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7144	uint32_t const uDr7 = pCtx->dr[7];
7145	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7146	&& X86_DR7_ANY_RW_IO(uDr7)
7147	&& (pCtx->cr4 & X86_CR4_DE))
7148	\|\| DBGFBpIsHwIoArmed(pVM)))
7149	{
7150	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7151	VMMRZCallRing3Disable(pVCpu);
7152	HM_DISABLE_PREEMPT();
7153
7154	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7155	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7156
7157	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, IoExitInfo.n.u16Port, cbValue);
7158	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7159	{
7160	/* Raise #DB. */
7161	pVmcb->guest.u64DR6 = pCtx->dr[6];
7162	pVmcb->guest.u64DR7 = pCtx->dr[7];
7163	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7164	hmR0SvmSetPendingXcptDB(pVCpu);
7165	}
7166	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7167	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7168	else if ( rcStrict2 != VINF_SUCCESS
7169	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7170	rcStrict = rcStrict2;
7171	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7172
7173	HM_RESTORE_PREEMPT();
7174	VMMRZCallRing3Enable(pVCpu);
7175	}
7176
7177	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7178	}
7179
7180	#ifdef VBOX_STRICT
7181	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
7182	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7183	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
7184	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7185	else
7186	{
7187	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7188	* statuses, that the VMM device and some others may return. See
7189	* IOM_SUCCESS() for guidance. */
7190	AssertMsg( RT_FAILURE(rcStrict)
7191	\|\| rcStrict == VINF_SUCCESS
7192	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7193	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7194	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7195	\|\| rcStrict == VINF_EM_RAW_TO_R3
7196	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED
7197	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7198	}
7199	#endif
7200	}
7201	else
7202	{
7203	/*
7204	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7205	*/
7206	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7207	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7208	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7209	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7210	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7211	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7212	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7213
7214	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7215	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7216
7217	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7218	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7219	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7220	}
7221	return VBOXSTRICTRC_TODO(rcStrict);
7222	}
7223
7224
7225	/**
7226	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7227	*/
7228	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7229	{
7230	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7231	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7232	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7233
7234	PVM pVM = pVCpu->CTX_SUFF(pVM);
7235	Assert(pVM->hm.s.fNestedPaging);
7236
7237	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7238	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7239	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7240	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7241	thus intentionally not copied into u32ErrCode. */
7242
7243	Log4Func(("#NPF at CS:RIP=%04x:%#RX64 GCPhysFaultAddr=%RGp ErrCode=%#x \n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7244	u32ErrCode));
7245
7246	/*
7247	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7248	*/
7249	if ( pVM->hm.s.fTprPatchingAllowed
7250	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7251	&& (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7252	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7253	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7254	&& !CPUMIsGuestInLongModeEx(pCtx)
7255	&& !CPUMGetGuestCPL(pVCpu)
7256	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7257	{
7258	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7259	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7260
7261	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7262	{
7263	/* Only attempt to patch the instruction once. */
7264	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7265	if (!pPatch)
7266	return VINF_EM_HM_PATCH_TPR_INSTR;
7267	}
7268	}
7269
7270	/*
7271	* Determine the nested paging mode.
7272	*/
7273	PGMMODE enmNestedPagingMode;
7274	#if HC_ARCH_BITS == 32
7275	if (CPUMIsGuestInLongModeEx(pCtx))
7276	enmNestedPagingMode = PGMMODE_AMD64_NX;
7277	else
7278	#endif
7279	enmNestedPagingMode = PGMGetHostMode(pVM);
7280
7281	/*
7282	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7283	*/
7284	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7285	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7286	{
7287	/*
7288	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7289	* injecting the original pending event would most likely cause the same MMIO #NPF.
7290	*/
7291	if (pVCpu->hm.s.Event.fPending)
7292	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7293
7294	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_RIP
7295	\| CPUMCTX_EXTRN_CS);
7296	VBOXSTRICTRC rcStrict;
7297	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7298	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7299	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7300	if (!pExitRec)
7301	{
7302
7303	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
7304	u32ErrCode);
7305
7306	/*
7307	* If we succeed, resume guest execution.
7308	*
7309	* If we fail in interpreting the instruction because we couldn't get the guest
7310	* physical address of the page containing the instruction via the guest's page
7311	* tables (we would invalidate the guest page in the host TLB), resume execution
7312	* which would cause a guest page fault to let the guest handle this weird case.
7313	*
7314	* See @bugref{6043}.
7315	*/
7316	if ( rcStrict == VINF_SUCCESS
7317	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7318	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
7319	{
7320	/* Successfully handled MMIO operation. */
7321	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7322	rcStrict = VINF_SUCCESS;
7323	}
7324	}
7325	else
7326	{
7327	/*
7328	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7329	*/
7330	Assert(pCtx == &pVCpu->cpum.GstCtx);
7331	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7332	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7333	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
7334
7335	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7336	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7337
7338	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7339	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7340	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7341	}
7342	return VBOXSTRICTRC_TODO(rcStrict);
7343	}
7344
7345	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
7346	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
7347	TRPMResetTrap(pVCpu);
7348
7349	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
7350
7351	/*
7352	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
7353	*/
7354	if ( rc == VINF_SUCCESS
7355	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7356	\|\| rc == VERR_PAGE_NOT_PRESENT)
7357	{
7358	/* We've successfully synced our shadow page tables. */
7359	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7360	rc = VINF_SUCCESS;
7361	}
7362
7363	return rc;
7364	}
7365
7366
7367	/**
7368	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
7369	* \#VMEXIT.
7370	*/
7371	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7372	{
7373	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7374	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
7375
7376	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7377	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7378	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb, pCtx);
7379
7380	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7382	return VINF_SUCCESS;
7383	}
7384
7385
7386	/**
7387	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
7388	* \#VMEXIT.
7389	*/
7390	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7391	{
7392	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7393	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7394
7395	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
7396	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
7397	#endif
7398
7399	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
7400	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
7401	{
7402	/*
7403	* AMD-V provides us with the exception which caused the TS; we collect
7404	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
7405	*/
7406	Log4Func(("TS occurred during event delivery\n"));
7407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7408	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7409	}
7410
7411	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7412	* emulation. */
7413	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7414	return VERR_EM_INTERPRETER;
7415	}
7416
7417
7418	/**
7419	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7420	*/
7421	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7422	{
7423	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7424	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7425
7426	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingAllowed)
7427	{
7428	int rc = hmSvmEmulateMovTpr(pVCpu, pCtx);
7429	if (rc != VERR_NOT_FOUND)
7430	{
7431	Log4Func(("hmSvmEmulateMovTpr returns %Rrc\n", rc));
7432	return rc;
7433	}
7434	}
7435
7436	if (EMAreHypercallInstructionsEnabled(pVCpu))
7437	{
7438	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, pCtx);
7439	if (RT_SUCCESS(rcStrict))
7440	{
7441	/* Only update the RIP if we're continuing guest execution and not in the case
7442	of say VINF_GIM_R3_HYPERCALL. */
7443	if (rcStrict == VINF_SUCCESS)
7444	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3 /* cbInstr */);
7445
7446	return VBOXSTRICTRC_VAL(rcStrict);
7447	}
7448	else
7449	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
7450	}
7451
7452	hmR0SvmSetPendingXcptUD(pVCpu);
7453	return VINF_SUCCESS;
7454	}
7455
7456
7457	/**
7458	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7459	*/
7460	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7461	{
7462	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7463	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
7464	/** @todo The guest has likely hit a contended spinlock. We might want to
7465	* poke a schedule different guest VCPU. */
7466	return VINF_EM_RAW_INTERRUPT;
7467	}
7468
7469
7470	/**
7471	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
7472	* \#VMEXIT.
7473	*/
7474	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7475	{
7476	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7477	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
7478	Assert(!(pCtx->cr0 & X86_CR0_NE));
7479
7480	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
7481	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
7482	}
7483
7484
7485	/**
7486	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
7487	*/
7488	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7489	{
7490	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7491
7492	/* Clear NMI blocking. */
7493	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
7494	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7495
7496	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7497	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7498	hmR0SvmClearCtrlIntercept(pVCpu, pCtx, pVmcb, SVM_CTRL_INTERCEPT_IRET);
7499
7500	/* Deliver the pending NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7501	return VINF_SUCCESS;
7502	}
7503
7504
7505	/**
7506	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
7507	* Conditional \#VMEXIT.
7508	*/
7509	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7510	{
7511	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7512	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7513	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7514
7515	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7516	PVM pVM = pVCpu->CTX_SUFF(pVM);
7517	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7518	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
7519	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7520
7521	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
7522	if (pVM->hm.s.fNestedPaging)
7523	{
7524	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7525	if ( !pSvmTransient->fVectoringDoublePF
7526	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7527	{
7528	/* A genuine guest #PF, reflect it to the guest. */
7529	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, uErrCode, uFaultAddress);
7530	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
7531	uFaultAddress, uErrCode));
7532	}
7533	else
7534	{
7535	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7536	hmR0SvmSetPendingXcptDF(pVCpu);
7537	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
7538	}
7539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7540	return VINF_SUCCESS;
7541	}
7542	#endif
7543
7544	Assert(!pVM->hm.s.fNestedPaging);
7545
7546	/*
7547	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
7548	*/
7549	if ( pVM->hm.s.fTprPatchingAllowed
7550	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
7551	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
7552	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7553	&& !CPUMIsGuestInLongModeEx(pCtx)
7554	&& !CPUMGetGuestCPL(pVCpu)
7555	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7556	{
7557	RTGCPHYS GCPhysApicBase;
7558	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7559	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7560
7561	/* Check if the page at the fault-address is the APIC base. */
7562	RTGCPHYS GCPhysPage;
7563	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
7564	if ( rc2 == VINF_SUCCESS
7565	&& GCPhysPage == GCPhysApicBase)
7566	{
7567	/* Only attempt to patch the instruction once. */
7568	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7569	if (!pPatch)
7570	return VINF_EM_HM_PATCH_TPR_INSTR;
7571	}
7572	}
7573
7574	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7575	pCtx->rip, uErrCode, pCtx->cr3));
7576
7577	/*
7578	* If it's a vectoring #PF, emulate injecting the original event injection as
7579	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
7580	* event injection that caused a #PF. See @bugref{6607}.
7581	*/
7582	if (pSvmTransient->fVectoringPF)
7583	{
7584	Assert(pVCpu->hm.s.Event.fPending);
7585	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7586	}
7587
7588	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
7589	int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7590
7591	Log4Func(("#PF: rc=%Rrc\n", rc));
7592
7593	if (rc == VINF_SUCCESS)
7594	{
7595	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7596	TRPMResetTrap(pVCpu);
7597	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7598	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7599	return rc;
7600	}
7601
7602	if (rc == VINF_EM_RAW_GUEST_TRAP)
7603	{
7604	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7605
7606	/*
7607	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
7608	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
7609	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
7610	*/
7611	if ( !pSvmTransient->fVectoringDoublePF
7612	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7613	{
7614	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
7615	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7616	TRPMResetTrap(pVCpu);
7617
7618	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7619	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
7620	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7621	&& HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_PF))
7622	return VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress));
7623	#endif
7624
7625	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, uErrCode, uFaultAddress);
7626	}
7627	else
7628	{
7629	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7630	TRPMResetTrap(pVCpu);
7631	hmR0SvmSetPendingXcptDF(pVCpu);
7632	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
7633	}
7634
7635	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7636	return VINF_SUCCESS;
7637	}
7638
7639	TRPMResetTrap(pVCpu);
7640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7641	return rc;
7642	}
7643
7644
7645	/**
7646	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
7647	* Conditional \#VMEXIT.
7648	*/
7649	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7650	{
7651	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7652	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
7653
7654	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7655	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7656	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7657
7658	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
7659	if (pVCpu->hm.s.fGIMTrapXcptUD)
7660	{
7661	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7662	uint8_t cbInstr = 0;
7663	VBOXSTRICTRC rcStrict = GIMXcptUD(pVCpu, pCtx, NULL /* pDis */, &cbInstr);
7664	if (rcStrict == VINF_SUCCESS)
7665	{
7666	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
7667	hmR0SvmAdvanceRipDumb(pVCpu, pCtx, cbInstr);
7668	rc = VINF_SUCCESS;
7669	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7670	}
7671	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
7672	rc = VINF_SUCCESS;
7673	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
7674	rc = VINF_GIM_R3_HYPERCALL;
7675	else
7676	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
7677	}
7678
7679	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
7680	if (RT_FAILURE(rc))
7681	{
7682	hmR0SvmSetPendingXcptUD(pVCpu);
7683	rc = VINF_SUCCESS;
7684	}
7685
7686	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7687	return rc;
7688	}
7689
7690
7691	/**
7692	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
7693	* Conditional \#VMEXIT.
7694	*/
7695	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7696	{
7697	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7698	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7699
7700	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7701	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7702	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7703
7704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
7705
7706	if (!(pCtx->cr0 & X86_CR0_NE))
7707	{
7708	PVM pVM = pVCpu->CTX_SUFF(pVM);
7709	PDISSTATE pDis = &pVCpu->hm.s.DisState;
7710	unsigned cbOp;
7711	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
7712	if (RT_SUCCESS(rc))
7713	{
7714	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
7715	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
7716	if (RT_SUCCESS(rc))
7717	pCtx->rip += cbOp;
7718	}
7719	else
7720	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
7721	return rc;
7722	}
7723
7724	hmR0SvmSetPendingXcptMF(pVCpu);
7725	return VINF_SUCCESS;
7726	}
7727
7728
7729	/**
7730	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
7731	* \#VMEXIT.
7732	*/
7733	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7734	{
7735	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7736	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7737	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7738
7739	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7740	{
7741	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7742	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7743	}
7744
7745	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
7746
7747	/*
7748	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
7749	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
7750	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
7751	*/
7752	PVM pVM = pVCpu->CTX_SUFF(pVM);
7753	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7754	int rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
7755	if (rc == VINF_EM_RAW_GUEST_TRAP)
7756	{
7757	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
7758	if (CPUMIsHyperDebugStateActive(pVCpu))
7759	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
7760
7761	/* Reflect the exception back to the guest. */
7762	hmR0SvmSetPendingXcptDB(pVCpu);
7763	rc = VINF_SUCCESS;
7764	}
7765
7766	/*
7767	* Update DR6.
7768	*/
7769	if (CPUMIsHyperDebugStateActive(pVCpu))
7770	{
7771	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
7772	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
7773	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7774	}
7775	else
7776	{
7777	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
7778	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
7779	}
7780
7781	return rc;
7782	}
7783
7784
7785	/**
7786	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
7787	* Conditional \#VMEXIT.
7788	*/
7789	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7790	{
7791	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7792	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7793
7794	SVMEVENT Event;
7795	Event.u = 0;
7796	Event.n.u1Valid = 1;
7797	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7798	Event.n.u8Vector = X86_XCPT_AC;
7799	Event.n.u1ErrorCodeValid = 1;
7800	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7801	return VINF_SUCCESS;
7802	}
7803
7804
7805	/**
7806	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
7807	* Conditional \#VMEXIT.
7808	*/
7809	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7810	{
7811	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7812	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7813	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7814
7815	int rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7816	if (rc == VINF_EM_RAW_GUEST_TRAP)
7817	{
7818	SVMEVENT Event;
7819	Event.u = 0;
7820	Event.n.u1Valid = 1;
7821	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7822	Event.n.u8Vector = X86_XCPT_BP;
7823	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7824	}
7825
7826	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
7827	return rc;
7828	}
7829
7830
7831	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM)
7832	/**
7833	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
7834	*/
7835	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7836	{
7837	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7838	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7839
7840	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7841	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
7842	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
7843	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
7844	Assert(uVector <= X86_XCPT_LAST);
7845	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
7846
7847	SVMEVENT Event;
7848	Event.u = 0;
7849	Event.n.u1Valid = 1;
7850	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7851	Event.n.u8Vector = uVector;
7852	switch (uVector)
7853	{
7854	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
7855	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
7856	case X86_XCPT_DF:
7857	case X86_XCPT_TS:
7858	case X86_XCPT_NP:
7859	case X86_XCPT_SS:
7860	case X86_XCPT_GP:
7861	case X86_XCPT_AC:
7862	{
7863	Event.n.u1ErrorCodeValid = 1;
7864	Event.n.u32ErrorCode = uErrCode;
7865	break;
7866	}
7867	}
7868
7869	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7870	return VINF_SUCCESS;
7871	}
7872	#endif
7873
7874	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7875	/**
7876	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
7877	*/
7878	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7879	{
7880	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7881	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
7882	\| CPUMCTX_EXTRN_HWVIRT);
7883
7884	#ifdef VBOX_STRICT
7885	PCSVMVMCB pVmcbTmp = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7886	Assert(pVmcbTmp);
7887	Assert(!pVmcbTmp->ctrl.IntCtrl.n.u1VGifEnable);
7888	RT_NOREF(pVmcbTmp);
7889	#endif
7890
7891	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7892	VBOXSTRICTRC rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
7893	if (rcStrict == VINF_SUCCESS)
7894	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
7895	return VBOXSTRICTRC_VAL(rcStrict);
7896	}
7897
7898
7899	/**
7900	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
7901	*/
7902	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7903	{
7904	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7905	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
7906	\| CPUMCTX_EXTRN_HWVIRT);
7907
7908	/*
7909	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
7910	* we only intercept STGI when events are pending for GIF to become 1.
7911	*/
7912	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7913	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
7914	hmR0SvmClearCtrlIntercept(pVCpu, pCtx, pVmcb, SVM_CTRL_INTERCEPT_STGI);
7915
7916	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7917	VBOXSTRICTRC rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
7918	if (rcStrict == VINF_SUCCESS)
7919	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
7920	return VBOXSTRICTRC_VAL(rcStrict);
7921	}
7922
7923
7924	/**
7925	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
7926	*/
7927	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7928	{
7929	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7930	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
7931	\| CPUMCTX_EXTRN_FS
7932	\| CPUMCTX_EXTRN_GS
7933	\| CPUMCTX_EXTRN_TR
7934	\| CPUMCTX_EXTRN_LDTR
7935	\| CPUMCTX_EXTRN_KERNEL_GS_BASE
7936	\| CPUMCTX_EXTRN_SYSCALL_MSRS
7937	\| CPUMCTX_EXTRN_SYSENTER_MSRS);
7938
7939	#ifdef VBOX_STRICT
7940	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7941	Assert(pVmcb);
7942	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7943	RT_NOREF(pVmcb);
7944	#endif
7945
7946	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7947	VBOXSTRICTRC rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
7948	if (rcStrict == VINF_SUCCESS)
7949	{
7950	/* We skip flagging changes made to LSTAR, STAR, SFMASK and other MSRs as they are always re-loaded. */
7951	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS
7952	\| HM_CHANGED_GUEST_GS
7953	\| HM_CHANGED_GUEST_TR
7954	\| HM_CHANGED_GUEST_LDTR
7955	\| HM_CHANGED_GUEST_SYSENTER_CS_MSR
7956	\| HM_CHANGED_GUEST_SYSENTER_EIP_MSR
7957	\| HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
7958	}
7959	return VBOXSTRICTRC_VAL(rcStrict);
7960	}
7961
7962
7963	/**
7964	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
7965	*/
7966	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7967	{
7968	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7969	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7970
7971	#ifdef VBOX_STRICT
7972	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7973	Assert(pVmcb);
7974	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7975	RT_NOREF(pVmcb);
7976	#endif
7977
7978	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7979	VBOXSTRICTRC rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
7980	return VBOXSTRICTRC_VAL(rcStrict);
7981	}
7982
7983
7984	/**
7985	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
7986	*/
7987	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7988	{
7989	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7990	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7991
7992	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7993	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
7994	return VBOXSTRICTRC_VAL(rcStrict);
7995	}
7996
7997
7998	/**
7999	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
8000	*/
8001	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
8002	{
8003	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
8004	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8005	\| IEM_CPUMCTX_EXTRN_SVM_VMRUN_MASK);
8006	VBOXSTRICTRC rcStrict;
8007	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
8008	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
8009	Log4Func(("IEMExecDecodedVmrun returns %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8010	if (rcStrict == VINF_SUCCESS)
8011	{
8012	rcStrict = VINF_SVM_VMRUN;
8013	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
8014	}
8015	return VBOXSTRICTRC_VAL(rcStrict);
8016	}
8017
8018
8019	/**
8020	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
8021	* Unconditional \#VMEXIT.
8022	*/
8023	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
8024	{
8025	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
8026	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
8027
8028	if (pVCpu->hm.s.Event.fPending)
8029	{
8030	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
8031	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8032	}
8033
8034	hmR0SvmSetPendingXcptDB(pVCpu);
8035	return VINF_SUCCESS;
8036	}
8037
8038
8039	/**
8040	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8041	* Conditional \#VMEXIT.
8042	*/
8043	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
8044	{
8045	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
8046	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
8047
8048	SVMEVENT Event;
8049	Event.u = 0;
8050	Event.n.u1Valid = 1;
8051	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8052	Event.n.u8Vector = X86_XCPT_BP;
8053	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8054	return VINF_SUCCESS;
8055	}
8056
8057	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
8058
8059
8060	/** @} */
8061

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 72777

Download in other formats: