HMSVMR0.cpp@ 75649

Last change on this file since 75649 was 75611, checked in by vboxsync, 6 years ago
VMM: Nested VMX: bugref:9180 Move the VMX APIC-access guest-physical page registration into IEM and got rid of the CPUM all context code that does not quite fit because we still have to declare the prototypes in the HM headers anyway, so just keep it in HM all context code for now.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 319.3 KB

Line
1	/* $Id: HMSVMR0.cpp 75611 2018-11-20 11:20:25Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#include "HMInternal.h"
36	#include <VBox/vmm/vm.h>
37	#include "HMSVMR0.h"
38	#include "dtrace/VBoxVMM.h"
39
40	#ifdef DEBUG_ramshankar
41	# define HMSVM_SYNC_FULL_GUEST_STATE
42	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
43	# define HMSVM_ALWAYS_TRAP_PF
44	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
45	#endif
46
47
48	/*********************************************************************************************************************************
49	* Defined Constants And Macros *
50	*********************************************************************************************************************************/
51	#ifdef VBOX_WITH_STATISTICS
52	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
53	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
54	if ((u64ExitCode) == SVM_EXIT_NPF) \
55	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
56	else \
57	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
58	} while (0)
59
60	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
61	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
62	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
63	if ((u64ExitCode) == SVM_EXIT_NPF) \
64	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
65	else \
66	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
67	} while (0)
68	# endif
69	#else
70	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
71	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
72	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
73	# endif
74	#endif /* !VBOX_WITH_STATISTICS */
75
76	/** If we decide to use a function table approach this can be useful to
77	* switch to a "static DECLCALLBACK(int)". */
78	#define HMSVM_EXIT_DECL static int
79
80	/**
81	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
82	* guest using hardware-assisted SVM.
83	*
84	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
85	* are swapped and restored across the world-switch and also registers like
86	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
87	* \#VMEXIT.
88	*/
89	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
90	\| CPUMCTX_EXTRN_RFLAGS \
91	\| CPUMCTX_EXTRN_RAX \
92	\| CPUMCTX_EXTRN_RSP \
93	\| CPUMCTX_EXTRN_SREG_MASK \
94	\| CPUMCTX_EXTRN_CR0 \
95	\| CPUMCTX_EXTRN_CR2 \
96	\| CPUMCTX_EXTRN_CR3 \
97	\| CPUMCTX_EXTRN_TABLE_MASK \
98	\| CPUMCTX_EXTRN_DR6 \
99	\| CPUMCTX_EXTRN_DR7 \
100	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
101	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
102	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
103	\| CPUMCTX_EXTRN_HWVIRT \
104	\| CPUMCTX_EXTRN_HM_SVM_MASK)
105
106	/**
107	* Subset of the guest-CPU state that is shared between the guest and host.
108	*/
109	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
110
111	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
112	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
113	do { \
114	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
115	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
116	} while (0)
117
118	/** Assert that the required state bits are fetched. */
119	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
120	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
121	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
122
123	/** Assert that preemption is disabled or covered by thread-context hooks. */
124	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
125	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
126
127	/** Assert that we haven't migrated CPUs when thread-context hooks are not
128	* used. */
129	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
130	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
131	("Illegal migration! Entered on CPU %u Current %u\n", \
132	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()));
133
134	/** Assert that we're not executing a nested-guest. */
135	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
136	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
137	#else
138	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
139	#endif
140
141	/** Assert that we're executing a nested-guest. */
142	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
143	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
144	#else
145	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
146	#endif
147
148	/** Macro for checking and returning from the using function for
149	* \#VMEXIT intercepts that maybe caused during delivering of another
150	* event in the guest. */
151	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
152	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
153	do \
154	{ \
155	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
156	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
157	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
158	else if ( rc == VINF_EM_RESET \
159	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
160	{ \
161	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
162	return VBOXSTRICTRC_TODO(IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0)); \
163	} \
164	else \
165	return rc; \
166	} while (0)
167	#else
168	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
169	do \
170	{ \
171	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
172	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
173	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
174	else \
175	return rc; \
176	} while (0)
177	#endif
178
179	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
180	* instruction that exited. */
181	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
182	do { \
183	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
184	(a_rc) = VINF_EM_DBG_STEPPED; \
185	} while (0)
186
187	/** Validate segment descriptor granularity bit. */
188	#ifdef VBOX_STRICT
189	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
190	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
191	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
192	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
193	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
194	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
195	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
196	#else
197	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
198	#endif
199
200	/**
201	* Exception bitmap mask for all contributory exceptions.
202	*
203	* Page fault is deliberately excluded here as it's conditional as to whether
204	* it's contributory or benign. Page faults are handled separately.
205	*/
206	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
207	\| RT_BIT(X86_XCPT_DE))
208
209	/**
210	* Mandatory/unconditional guest control intercepts.
211	*
212	* SMIs can and do happen in normal operation. We need not intercept them
213	* while executing the guest (or nested-guest).
214	*/
215	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
216	\| SVM_CTRL_INTERCEPT_NMI \
217	\| SVM_CTRL_INTERCEPT_INIT \
218	\| SVM_CTRL_INTERCEPT_RDPMC \
219	\| SVM_CTRL_INTERCEPT_CPUID \
220	\| SVM_CTRL_INTERCEPT_RSM \
221	\| SVM_CTRL_INTERCEPT_HLT \
222	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
223	\| SVM_CTRL_INTERCEPT_MSR_PROT \
224	\| SVM_CTRL_INTERCEPT_INVLPGA \
225	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
226	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
227	\| SVM_CTRL_INTERCEPT_VMRUN \
228	\| SVM_CTRL_INTERCEPT_SKINIT \
229	\| SVM_CTRL_INTERCEPT_WBINVD \
230	\| SVM_CTRL_INTERCEPT_MONITOR \
231	\| SVM_CTRL_INTERCEPT_MWAIT \
232	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
233	\| SVM_CTRL_INTERCEPT_XSETBV)
234
235	/** @name VMCB Clean Bits.
236	*
237	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
238	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
239	* memory.
240	*
241	* @{ */
242	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
243	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
244	/** I/O permission bitmap, MSR permission bitmap. */
245	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
246	/** ASID. */
247	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
248	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
249	V_INTR_VECTOR. */
250	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
251	/** Nested Paging: Nested CR3 (nCR3), PAT. */
252	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
253	/** Control registers (CR0, CR3, CR4, EFER). */
254	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
255	/** Debug registers (DR6, DR7). */
256	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
257	/** GDT, IDT limit and base. */
258	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
259	/** Segment register: CS, SS, DS, ES limit and base. */
260	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
261	/** CR2.*/
262	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
263	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
264	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
265	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
266	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
267	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
268	/** Mask of all valid VMCB Clean bits. */
269	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
270	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
271	\| HMSVM_VMCB_CLEAN_ASID \
272	\| HMSVM_VMCB_CLEAN_INT_CTRL \
273	\| HMSVM_VMCB_CLEAN_NP \
274	\| HMSVM_VMCB_CLEAN_CRX_EFER \
275	\| HMSVM_VMCB_CLEAN_DRX \
276	\| HMSVM_VMCB_CLEAN_DT \
277	\| HMSVM_VMCB_CLEAN_SEG \
278	\| HMSVM_VMCB_CLEAN_CR2 \
279	\| HMSVM_VMCB_CLEAN_LBR \
280	\| HMSVM_VMCB_CLEAN_AVIC)
281	/** @} */
282
283	/** @name SVM transient.
284	*
285	* A state structure for holding miscellaneous information across AMD-V
286	* VMRUN/\#VMEXIT operation, restored after the transition.
287	*
288	* @{ */
289	typedef struct SVMTRANSIENT
290	{
291	/** The host's rflags/eflags. */
292	RTCCUINTREG fEFlags;
293	#if HC_ARCH_BITS == 32
294	uint32_t u32Alignment0;
295	#endif
296
297	/** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
298	uint64_t u64ExitCode;
299	/** The guest's TPR value used for TPR shadowing. */
300	uint8_t u8GuestTpr;
301	/** Alignment. */
302	uint8_t abAlignment0[7];
303
304	/** Pointer to the currently executing VMCB. */
305	PSVMVMCB pVmcb;
306	/** Whether we are currently executing a nested-guest. */
307	bool fIsNestedGuest;
308
309	/** Whether the guest debug state was active at the time of \#VMEXIT. */
310	bool fWasGuestDebugStateActive;
311	/** Whether the hyper debug state was active at the time of \#VMEXIT. */
312	bool fWasHyperDebugStateActive;
313	/** Whether the TSC offset mode needs to be updated. */
314	bool fUpdateTscOffsetting;
315	/** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
316	bool fRestoreTscAuxMsr;
317	/** Whether the \#VMEXIT was caused by a page-fault during delivery of a
318	* contributary exception or a page-fault. */
319	bool fVectoringDoublePF;
320	/** Whether the \#VMEXIT was caused by a page-fault during delivery of an
321	* external interrupt or NMI. */
322	bool fVectoringPF;
323	} SVMTRANSIENT, *PSVMTRANSIENT;
324	AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
325	AssertCompileMemberAlignment(SVMTRANSIENT, pVmcb, sizeof(uint64_t));
326	/** @} */
327
328	/**
329	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
330	*/
331	typedef enum SVMMSREXITREAD
332	{
333	/** Reading this MSR causes a \#VMEXIT. */
334	SVMMSREXIT_INTERCEPT_READ = 0xb,
335	/** Reading this MSR does not cause a \#VMEXIT. */
336	SVMMSREXIT_PASSTHRU_READ
337	} SVMMSREXITREAD;
338
339	/**
340	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
341	*/
342	typedef enum SVMMSREXITWRITE
343	{
344	/** Writing to this MSR causes a \#VMEXIT. */
345	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
346	/** Writing to this MSR does not cause a \#VMEXIT. */
347	SVMMSREXIT_PASSTHRU_WRITE
348	} SVMMSREXITWRITE;
349
350	/**
351	* SVM \#VMEXIT handler.
352	*
353	* @returns VBox status code.
354	* @param pVCpu The cross context virtual CPU structure.
355	* @param pSvmTransient Pointer to the SVM-transient structure.
356	*/
357	typedef int FNSVMEXITHANDLER(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient);
358
359
360	/*********************************************************************************************************************************
361	* Internal Functions *
362	*********************************************************************************************************************************/
363	static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu);
364	static void hmR0SvmLeave(PVMCPU pVCpu, bool fImportState);
365
366
367	/** @name \#VMEXIT handlers.
368	* @{
369	*/
370	static FNSVMEXITHANDLER hmR0SvmExitIntr;
371	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
372	static FNSVMEXITHANDLER hmR0SvmExitInvd;
373	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
374	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
375	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
376	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
377	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
378	static FNSVMEXITHANDLER hmR0SvmExitHlt;
379	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
380	static FNSVMEXITHANDLER hmR0SvmExitMwait;
381	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
382	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
383	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
384	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
385	static FNSVMEXITHANDLER hmR0SvmExitMsr;
386	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
387	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
388	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
389	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
390	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
391	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
392	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
393	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
394	static FNSVMEXITHANDLER hmR0SvmExitPause;
395	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
396	static FNSVMEXITHANDLER hmR0SvmExitIret;
397	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
398	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
399	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
400	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
401	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
402	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
403	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM)
404	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
405	#endif
406	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
407	static FNSVMEXITHANDLER hmR0SvmExitClgi;
408	static FNSVMEXITHANDLER hmR0SvmExitStgi;
409	static FNSVMEXITHANDLER hmR0SvmExitVmload;
410	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
411	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
412	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
413	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
414	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
415	#endif
416	/** @} */
417
418	static int hmR0SvmHandleExit(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient);
419	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
420	static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient);
421	#endif
422
423
424	/*********************************************************************************************************************************
425	* Global Variables *
426	*********************************************************************************************************************************/
427	/** Ring-0 memory object for the IO bitmap. */
428	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
429	/** Physical address of the IO bitmap. */
430	static RTHCPHYS g_HCPhysIOBitmap;
431	/** Pointer to the IO bitmap. */
432	static R0PTRTYPE(void *) g_pvIOBitmap;
433
434	#ifdef VBOX_STRICT
435	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
436	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
437	# define HMSVM_LOG_CS RT_BIT_32(2)
438	# define HMSVM_LOG_SS RT_BIT_32(3)
439	# define HMSVM_LOG_FS RT_BIT_32(4)
440	# define HMSVM_LOG_GS RT_BIT_32(5)
441	# define HMSVM_LOG_LBR RT_BIT_32(6)
442	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
443	\| HMSVM_LOG_CR_REGS \
444	\| HMSVM_LOG_CS \
445	\| HMSVM_LOG_SS \
446	\| HMSVM_LOG_FS \
447	\| HMSVM_LOG_GS \
448	\| HMSVM_LOG_LBR)
449
450	/**
451	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
452	*
453	* @param pVCpu The cross context virtual CPU structure.
454	* @param pVmcb Pointer to the VM control block.
455	* @param pszPrefix Log prefix.
456	* @param fFlags Log flags, see HMSVM_LOG_XXX.
457	* @param uVerbose The verbosity level, currently unused.
458	*/
459	static void hmR0SvmLogState(PVMCPU pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
460	{
461	RT_NOREF2(pVCpu, uVerbose);
462	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
463
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
465	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
466
467	if (fFlags & HMSVM_LOG_RBP_RSP)
468	{
469	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
470	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
471	}
472
473	if (fFlags & HMSVM_LOG_CR_REGS)
474	{
475	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
476	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
477	}
478
479	if (fFlags & HMSVM_LOG_CS)
480	{
481	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
482	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
483	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
484	}
485	if (fFlags & HMSVM_LOG_SS)
486	{
487	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
488	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
489	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
490	}
491	if (fFlags & HMSVM_LOG_FS)
492	{
493	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
494	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
495	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
496	}
497	if (fFlags & HMSVM_LOG_GS)
498	{
499	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
500	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
501	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
502	}
503
504	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
505	if (fFlags & HMSVM_LOG_LBR)
506	{
507	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
508	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
509	}
510	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
511	}
512	#endif /* VBOX_STRICT */
513
514
515	/**
516	* Sets up and activates AMD-V on the current CPU.
517	*
518	* @returns VBox status code.
519	* @param pHostCpu Pointer to the CPU info struct.
520	* @param pVM The cross context VM structure. Can be
521	* NULL after a resume!
522	* @param pvCpuPage Pointer to the global CPU page.
523	* @param HCPhysCpuPage Physical address of the global CPU page.
524	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
525	* @param pvArg Unused on AMD-V.
526	*/
527	VMMR0DECL(int) SVMR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
528	void *pvArg)
529	{
530	Assert(!fEnabledByHost);
531	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
532	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
533	Assert(pvCpuPage); NOREF(pvCpuPage);
534	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
535
536	NOREF(pvArg);
537	NOREF(fEnabledByHost);
538
539	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
540	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
541
542	/*
543	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
544	*/
545	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
546	if (u64HostEfer & MSR_K6_EFER_SVME)
547	{
548	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
549	if ( pVM
550	&& pVM->hm.s.svm.fIgnoreInUseError)
551	pHostCpu->fIgnoreAMDVInUseError = true;
552
553	if (!pHostCpu->fIgnoreAMDVInUseError)
554	{
555	ASMSetFlags(fEFlags);
556	return VERR_SVM_IN_USE;
557	}
558	}
559
560	/* Turn on AMD-V in the EFER MSR. */
561	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
562
563	/* Write the physical page address where the CPU will store the host state while executing the VM. */
564	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
565
566	/* Restore interrupts. */
567	ASMSetFlags(fEFlags);
568
569	/*
570	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
571	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
572	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
573	* entirely with before executing any guest code.
574	*/
575	pHostCpu->fFlushAsidBeforeUse = true;
576
577	/*
578	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
579	*/
580	++pHostCpu->cTlbFlushes;
581
582	return VINF_SUCCESS;
583	}
584
585
586	/**
587	* Deactivates AMD-V on the current CPU.
588	*
589	* @returns VBox status code.
590	* @param pHostCpu Pointer to the CPU info struct.
591	* @param pvCpuPage Pointer to the global CPU page.
592	* @param HCPhysCpuPage Physical address of the global CPU page.
593	*/
594	VMMR0DECL(int) SVMR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
595	{
596	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
597	AssertReturn( HCPhysCpuPage
598	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
599	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
600	RT_NOREF(pHostCpu);
601
602	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
603	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
604
605	/* Turn off AMD-V in the EFER MSR. */
606	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
607	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
608
609	/* Invalidate host state physical address. */
610	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
611
612	/* Restore interrupts. */
613	ASMSetFlags(fEFlags);
614
615	return VINF_SUCCESS;
616	}
617
618
619	/**
620	* Does global AMD-V initialization (called during module initialization).
621	*
622	* @returns VBox status code.
623	*/
624	VMMR0DECL(int) SVMR0GlobalInit(void)
625	{
626	/*
627	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
628	* intercept all IO accesses, it's done once globally here instead of per-VM.
629	*/
630	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
631	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
632	if (RT_FAILURE(rc))
633	return rc;
634
635	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
636	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
637
638	/* Set all bits to intercept all IO accesses. */
639	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
640
641	return VINF_SUCCESS;
642	}
643
644
645	/**
646	* Does global AMD-V termination (called during module termination).
647	*/
648	VMMR0DECL(void) SVMR0GlobalTerm(void)
649	{
650	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
651	{
652	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
653	g_pvIOBitmap = NULL;
654	g_HCPhysIOBitmap = 0;
655	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
656	}
657	}
658
659
660	/**
661	* Frees any allocated per-VCPU structures for a VM.
662	*
663	* @param pVM The cross context VM structure.
664	*/
665	DECLINLINE(void) hmR0SvmFreeStructs(PVM pVM)
666	{
667	for (uint32_t i = 0; i < pVM->cCpus; i++)
668	{
669	PVMCPU pVCpu = &pVM->aCpus[i];
670	AssertPtr(pVCpu);
671
672	if (pVCpu->hm.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
673	{
674	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcbHost, false);
675	pVCpu->hm.s.svm.HCPhysVmcbHost = 0;
676	pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
677	}
678
679	if (pVCpu->hm.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
680	{
681	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcb, false);
682	pVCpu->hm.s.svm.pVmcb = NULL;
683	pVCpu->hm.s.svm.HCPhysVmcb = 0;
684	pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
685	}
686
687	if (pVCpu->hm.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
688	{
689	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjMsrBitmap, false);
690	pVCpu->hm.s.svm.pvMsrBitmap = NULL;
691	pVCpu->hm.s.svm.HCPhysMsrBitmap = 0;
692	pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
693	}
694	}
695	}
696
697
698	/**
699	* Does per-VM AMD-V initialization.
700	*
701	* @returns VBox status code.
702	* @param pVM The cross context VM structure.
703	*/
704	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
705	{
706	int rc = VERR_INTERNAL_ERROR_5;
707
708	/*
709	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
710	*/
711	uint32_t u32Family;
712	uint32_t u32Model;
713	uint32_t u32Stepping;
714	if (HMSvmIsSubjectToErratum170(&u32Family, &u32Model, &u32Stepping))
715	{
716	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
717	pVM->hm.s.svm.fAlwaysFlushTLB = true;
718	}
719
720	/*
721	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
722	*/
723	for (VMCPUID i = 0; i < pVM->cCpus; i++)
724	{
725	PVMCPU pVCpu = &pVM->aCpus[i];
726	pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
727	pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
728	pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
729	}
730
731	for (VMCPUID i = 0; i < pVM->cCpus; i++)
732	{
733	PVMCPU pVCpu = &pVM->aCpus[i];
734
735	/*
736	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
737	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
738	*/
739	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
740	if (RT_FAILURE(rc))
741	goto failure_cleanup;
742
743	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcbHost);
744	pVCpu->hm.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcbHost, 0 /* iPage */);
745	Assert(pVCpu->hm.s.svm.HCPhysVmcbHost < _4G);
746	ASMMemZeroPage(pvVmcbHost);
747
748	/*
749	* Allocate one page for the guest-state VMCB.
750	*/
751	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
752	if (RT_FAILURE(rc))
753	goto failure_cleanup;
754
755	pVCpu->hm.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcb);
756	pVCpu->hm.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcb, 0 /* iPage */);
757	Assert(pVCpu->hm.s.svm.HCPhysVmcb < _4G);
758	ASMMemZeroPage(pVCpu->hm.s.svm.pVmcb);
759
760	/*
761	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
762	* SVM to not require one.
763	*/
764	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
765	false /* fExecutable */);
766	if (RT_FAILURE(rc))
767	goto failure_cleanup;
768
769	pVCpu->hm.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjMsrBitmap);
770	pVCpu->hm.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
771	/* Set all bits to intercept all MSR accesses (changed later on). */
772	ASMMemFill32(pVCpu->hm.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
773	}
774
775	return VINF_SUCCESS;
776
777	failure_cleanup:
778	hmR0SvmFreeStructs(pVM);
779	return rc;
780	}
781
782
783	/**
784	* Does per-VM AMD-V termination.
785	*
786	* @returns VBox status code.
787	* @param pVM The cross context VM structure.
788	*/
789	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
790	{
791	hmR0SvmFreeStructs(pVM);
792	return VINF_SUCCESS;
793	}
794
795
796	/**
797	* Returns whether the VMCB Clean Bits feature is supported.
798	*
799	* @return @c true if supported, @c false otherwise.
800	* @param pVCpu The cross context virtual CPU structure.
801	*/
802	DECLINLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPU pVCpu)
803	{
804	PVM pVM = pVCpu->CTX_SUFF(pVM);
805	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
806	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
807	{
808	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN)
809	&& pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
810	}
811	#endif
812	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
813	}
814
815
816	/**
817	* Returns whether the decode assists feature is supported.
818	*
819	* @return @c true if supported, @c false otherwise.
820	* @param pVCpu The cross context virtual CPU structure.
821	*/
822	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPU pVCpu)
823	{
824	PVM pVM = pVCpu->CTX_SUFF(pVM);
825	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
826	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
827	{
828	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
829	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
830	}
831	#endif
832	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
833	}
834
835
836	/**
837	* Returns whether the NRIP_SAVE feature is supported.
838	*
839	* @return @c true if supported, @c false otherwise.
840	* @param pVCpu The cross context virtual CPU structure.
841	*/
842	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPU pVCpu)
843	{
844	PVM pVM = pVCpu->CTX_SUFF(pVM);
845	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
846	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
847	{
848	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
849	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
850	}
851	#endif
852	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
853	}
854
855
856	/**
857	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
858	*
859	* @param pVCpu The cross context virtual CPU structure.
860	* @param pbMsrBitmap Pointer to the MSR bitmap.
861	* @param idMsr The MSR for which the permissions are being set.
862	* @param enmRead MSR read permissions.
863	* @param enmWrite MSR write permissions.
864	*
865	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
866	* caller needs to take care of this.
867	*/
868	static void hmR0SvmSetMsrPermission(PVMCPU pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
869	SVMMSREXITWRITE enmWrite)
870	{
871	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
872	uint16_t offMsrpm;
873	uint8_t uMsrpmBit;
874	int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
875	AssertRC(rc);
876
877	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
878	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
879
880	pbMsrBitmap += offMsrpm;
881	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
882	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
883	else
884	{
885	if (!fInNestedGuestMode)
886	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
887	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
888	else
889	{
890	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
891	uint8_t const pbNstGstMsrBitmap = (uint8_t )pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
892	pbNstGstMsrBitmap += offMsrpm;
893	if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit)))
894	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
895	else
896	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
897	}
898	#endif
899	}
900
901	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
902	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
903	else
904	{
905	if (!fInNestedGuestMode)
906	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
907	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
908	else
909	{
910	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
911	uint8_t const pbNstGstMsrBitmap = (uint8_t )pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
912	pbNstGstMsrBitmap += offMsrpm;
913	if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit + 1)))
914	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
915	else
916	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
917	}
918	#endif
919	}
920	}
921
922
923	/**
924	* Sets up AMD-V for the specified VM.
925	* This function is only called once per-VM during initalization.
926	*
927	* @returns VBox status code.
928	* @param pVM The cross context VM structure.
929	*/
930	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
931	{
932	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
933	AssertReturn(pVM, VERR_INVALID_PARAMETER);
934	Assert(pVM->hm.s.svm.fSupported);
935
936	bool const fPauseFilter = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
937	bool const fPauseFilterThreshold = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
938	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
939
940	bool const fLbrVirt = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
941	bool const fUseLbrVirt = fLbrVirt; /** @todo CFGM, IEM implementation etc. */
942
943	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
944	bool const fVirtVmsaveVmload = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
945	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && pVM->hm.s.fNestedPaging;
946
947	bool const fVGif = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);
948	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
949	#endif
950
951	PVMCPU pVCpu = &pVM->aCpus[0];
952	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
953	AssertMsgReturn(pVmcb, ("Invalid pVmcb for vcpu[0]\n"), VERR_SVM_INVALID_PVMCB);
954	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
955
956	/* Always trap #AC for reasons of security. */
957	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
958
959	/* Always trap #DB for reasons of security. */
960	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
961
962	/* Trap exceptions unconditionally (debug purposes). */
963	#ifdef HMSVM_ALWAYS_TRAP_PF
964	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
965	#endif
966	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
967	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
968	pVmcbCtrl->u32InterceptXcpt \|= 0
969	\| RT_BIT(X86_XCPT_BP)
970	\| RT_BIT(X86_XCPT_DE)
971	\| RT_BIT(X86_XCPT_NM)
972	\| RT_BIT(X86_XCPT_UD)
973	\| RT_BIT(X86_XCPT_NP)
974	\| RT_BIT(X86_XCPT_SS)
975	\| RT_BIT(X86_XCPT_GP)
976	\| RT_BIT(X86_XCPT_PF)
977	\| RT_BIT(X86_XCPT_MF)
978	;
979	#endif
980
981	/* Apply the exceptions intercepts needed by the GIM provider. */
982	if (pVCpu->hm.s.fGIMTrapXcptUD)
983	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
984
985	/* Set up unconditional intercepts and conditions. */
986	pVmcbCtrl->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
987	\| SVM_CTRL_INTERCEPT_VMMCALL;
988
989	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
990	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
991	#endif
992
993	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
994	/* Virtualized VMSAVE/VMLOAD. */
995	pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload = fUseVirtVmsaveVmload;
996	if (!fUseVirtVmsaveVmload)
997	{
998	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
999	\| SVM_CTRL_INTERCEPT_VMLOAD;
1000	}
1001
1002	/* Virtual GIF. */
1003	pVmcbCtrl->IntCtrl.n.u1VGifEnable = fUseVGif;
1004	if (!fUseVGif)
1005	{
1006	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
1007	\| SVM_CTRL_INTERCEPT_STGI;
1008	}
1009	#endif
1010
1011	/* CR4 writes must always be intercepted for tracking PGM mode changes. */
1012	pVmcbCtrl->u16InterceptWrCRx = RT_BIT(4);
1013
1014	/* Intercept all DRx reads and writes by default. Changed later on. */
1015	pVmcbCtrl->u16InterceptRdDRx = 0xffff;
1016	pVmcbCtrl->u16InterceptWrDRx = 0xffff;
1017
1018	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1019	pVmcbCtrl->IntCtrl.n.u1VIntrMasking = 1;
1020
1021	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1022	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1023	pVmcbCtrl->IntCtrl.n.u1IgnoreTPR = 1;
1024
1025	/* Set the IO permission bitmap physical addresses. */
1026	pVmcbCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1027
1028	/* LBR virtualization. */
1029	pVmcbCtrl->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1030
1031	/* The host ASID MBZ, for the guest start with 1. */
1032	pVmcbCtrl->TLBCtrl.n.u32ASID = 1;
1033
1034	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1035	pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging = pVM->hm.s.fNestedPaging;
1036
1037	/* Without Nested Paging, we need additionally intercepts. */
1038	if (!pVM->hm.s.fNestedPaging)
1039	{
1040	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1041	pVmcbCtrl->u16InterceptRdCRx \|= RT_BIT(3);
1042	pVmcbCtrl->u16InterceptWrCRx \|= RT_BIT(3);
1043
1044	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1045	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1046	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1047
1048	/* Page faults must be intercepted to implement shadow paging. */
1049	pVmcbCtrl->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1050	}
1051
1052	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1053	if (fUsePauseFilter)
1054	{
1055	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1056	pVmcbCtrl->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1057	if (fPauseFilterThreshold)
1058	pVmcbCtrl->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1059	pVmcbCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1060	}
1061
1062	/*
1063	* Setup the MSR permission bitmap.
1064	* The following MSRs are saved/restored automatically during the world-switch.
1065	* Don't intercept guest read/write accesses to these MSRs.
1066	*/
1067	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
1068	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1069	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1070	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1071	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1072	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1073	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1074	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1075	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1076	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1077	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1078	pVmcbCtrl->u64MSRPMPhysAddr = pVCpu->hm.s.svm.HCPhysMsrBitmap;
1079
1080	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1081	Assert(pVmcbCtrl->u32VmcbCleanBits == 0);
1082
1083	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1084	{
1085	PVMCPU pVCpuCur = &pVM->aCpus[i];
1086	PSVMVMCB pVmcbCur = pVM->aCpus[i].hm.s.svm.pVmcb;
1087	AssertMsgReturn(pVmcbCur, ("Invalid pVmcb for vcpu[%u]\n", i), VERR_SVM_INVALID_PVMCB);
1088	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1089
1090	/* Copy the VMCB control area. */
1091	memcpy(pVmcbCtrlCur, pVmcbCtrl, sizeof(*pVmcbCtrlCur));
1092
1093	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1094	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hm.s.svm.pvMsrBitmap;
1095	memcpy(pbMsrBitmapCur, pbMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1096	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hm.s.svm.HCPhysMsrBitmap;
1097
1098	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1099	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1100
1101	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1102	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu->hm.s.fGIMTrapXcptUD);
1103	}
1104
1105	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1106	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1107	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1108	#else
1109	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1110	#endif
1111	return VINF_SUCCESS;
1112	}
1113
1114
1115	/**
1116	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1117	*
1118	* @returns Pointer to the current context VMCB.
1119	* @param pVCpu The cross context virtual CPU structure.
1120	*/
1121	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPU pVCpu)
1122	{
1123	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1124	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1125	return pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
1126	#endif
1127	return pVCpu->hm.s.svm.pVmcb;
1128	}
1129
1130
1131	/**
1132	* Gets a pointer to the nested-guest VMCB cache.
1133	*
1134	* @returns Pointer to the nested-guest VMCB cache.
1135	* @param pVCpu The cross context virtual CPU structure.
1136	*/
1137	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPU pVCpu)
1138	{
1139	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1140	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1141	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1142	#else
1143	RT_NOREF(pVCpu);
1144	return NULL;
1145	#endif
1146	}
1147
1148
1149	/**
1150	* Invalidates a guest page by guest virtual address.
1151	*
1152	* @returns VBox status code.
1153	* @param pVCpu The cross context virtual CPU structure.
1154	* @param GCVirt Guest virtual address of the page to invalidate.
1155	*/
1156	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1157	{
1158	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1159
1160	bool const fFlushPending = pVCpu->CTX_SUFF(pVM)->hm.s.svm.fAlwaysFlushTLB \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1161
1162	/* Skip it if a TLB flush is already pending. */
1163	if (!fFlushPending)
1164	{
1165	Log4Func(("%#RGv\n", GCVirt));
1166
1167	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1168	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1169
1170	#if HC_ARCH_BITS == 32
1171	/* If we get a flush in 64-bit guest mode, then force a full TLB flush. INVLPGA takes only 32-bit addresses. */
1172	if (CPUMIsGuestInLongMode(pVCpu))
1173	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1174	else
1175	#endif
1176	{
1177	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1178	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1179	}
1180	}
1181	return VINF_SUCCESS;
1182	}
1183
1184
1185	/**
1186	* Flushes the appropriate tagged-TLB entries.
1187	*
1188	* @param pVCpu The cross context virtual CPU structure.
1189	* @param pVmcb Pointer to the VM control block.
1190	* @param pHostCpu Pointer to the HM host-CPU info.
1191	*/
1192	static void hmR0SvmFlushTaggedTlb(PVMCPU pVCpu, PSVMVMCB pVmcb, PHMGLOBALCPUINFO pHostCpu)
1193	{
1194	/*
1195	* Force a TLB flush for the first world switch if the current CPU differs from the one
1196	* we ran on last. This can happen both for start & resume due to long jumps back to
1197	* ring-3.
1198	*
1199	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1200	* correlation between it and the physical CPU.
1201	*
1202	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1203	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1204	*/
1205	bool fNewAsid = false;
1206	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1207	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
1208	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes
1209	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1210	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1211	#endif
1212	)
1213	{
1214	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1215	pVCpu->hm.s.fForceTLBFlush = true;
1216	fNewAsid = true;
1217	}
1218
1219	/* Set TLB flush state as checked until we return from the world switch. */
1220	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1221
1222	/* Check for explicit TLB flushes. */
1223	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1224	{
1225	pVCpu->hm.s.fForceTLBFlush = true;
1226	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1227	}
1228
1229	/*
1230	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1231	* This Host CPU requirement takes precedence.
1232	*/
1233	PVM pVM = pVCpu->CTX_SUFF(pVM);
1234	if (pVM->hm.s.svm.fAlwaysFlushTLB)
1235	{
1236	pHostCpu->uCurrentAsid = 1;
1237	pVCpu->hm.s.uCurrentAsid = 1;
1238	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1239	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
1240	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1241
1242	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1243	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1244	}
1245	else
1246	{
1247	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1248	if (pVCpu->hm.s.fForceTLBFlush)
1249	{
1250	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1251	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1252
1253	if (fNewAsid)
1254	{
1255	++pHostCpu->uCurrentAsid;
1256
1257	bool fHitASIDLimit = false;
1258	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1259	{
1260	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1261	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1262	fHitASIDLimit = true;
1263	}
1264
1265	if ( fHitASIDLimit
1266	\|\| pHostCpu->fFlushAsidBeforeUse)
1267	{
1268	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1269	pHostCpu->fFlushAsidBeforeUse = false;
1270	}
1271
1272	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1273	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
1274	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1275	}
1276	else
1277	{
1278	if (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1279	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1280	else
1281	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1282	}
1283
1284	pVCpu->hm.s.fForceTLBFlush = false;
1285	}
1286	}
1287
1288	/* Update VMCB with the ASID. */
1289	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hm.s.uCurrentAsid)
1290	{
1291	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentAsid;
1292	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1293	}
1294
1295	AssertMsg(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu,
1296	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hm.s.idLastCpu, pHostCpu->idCpu));
1297	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1298	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1299	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1300	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1301	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1302	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1303
1304	#ifdef VBOX_WITH_STATISTICS
1305	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1306	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1307	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1308	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1309	{
1310	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1311	}
1312	else
1313	{
1314	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1315	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1316	}
1317	#endif
1318	}
1319
1320
1321	/** @name 64-bit guest on 32-bit host OS helper functions.
1322	*
1323	* The host CPU is still 64-bit capable but the host OS is running in 32-bit
1324	* mode (code segment, paging). These wrappers/helpers perform the necessary
1325	* bits for the 32->64 switcher.
1326	*
1327	* @{ */
1328	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1329	/**
1330	* Prepares for and executes VMRUN (64-bit guests on a 32-bit host).
1331	*
1332	* @returns VBox status code.
1333	* @param HCPhysVmcbHost Physical address of host VMCB.
1334	* @param HCPhysVmcb Physical address of the VMCB.
1335	* @param pCtx Pointer to the guest-CPU context.
1336	* @param pVM The cross context VM structure.
1337	* @param pVCpu The cross context virtual CPU structure.
1338	*/
1339	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
1340	{
1341	RT_NOREF2(pVM, pCtx);
1342	uint32_t aParam[8];
1343	aParam[0] = RT_LO_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Lo. */
1344	aParam[1] = RT_HI_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Hi. */
1345	aParam[2] = RT_LO_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Lo. */
1346	aParam[3] = RT_HI_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Hi. */
1347	aParam[4] = VM_RC_ADDR(pVM, pVM);
1348	aParam[5] = 0;
1349	aParam[6] = VM_RC_ADDR(pVM, pVCpu);
1350	aParam[7] = 0;
1351
1352	return SVMR0Execute64BitsHandler(pVCpu, HM64ON32OP_SVMRCVMRun64, RT_ELEMENTS(aParam), &aParam[0]);
1353	}
1354
1355
1356	/**
1357	* Executes the specified VMRUN handler in 64-bit mode.
1358	*
1359	* @returns VBox status code.
1360	* @param pVCpu The cross context virtual CPU structure.
1361	* @param enmOp The operation to perform.
1362	* @param cParams Number of parameters.
1363	* @param paParam Array of 32-bit parameters.
1364	*/
1365	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
1366	{
1367	PVM pVM = pVCpu->CTX_SUFF(pVM);
1368	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
1369	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
1370
1371	/* Disable interrupts. */
1372	RTHCUINTREG const fEFlags = ASMIntDisableFlags();
1373
1374	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1375	RTCPUID idHostCpu = RTMpCpuId();
1376	CPUMR0SetLApic(pVCpu, idHostCpu);
1377	#endif
1378
1379	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
1380	CPUMSetHyperEIP(pVCpu, enmOp);
1381	for (int i = (int)cParams - 1; i >= 0; i--)
1382	CPUMPushHyper(pVCpu, paParam[i]);
1383
1384	/* Call the switcher. */
1385	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
1386	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
1387	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
1388
1389	/* Restore interrupts. */
1390	ASMSetFlags(fEFlags);
1391	return rc;
1392	}
1393
1394	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
1395	/** @} */
1396
1397
1398	/**
1399	* Sets an exception intercept in the specified VMCB.
1400	*
1401	* @param pVmcb Pointer to the VM control block.
1402	* @param uXcpt The exception (X86_XCPT_*).
1403	*/
1404	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1405	{
1406	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1407	{
1408	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1409	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1410	}
1411	}
1412
1413
1414	/**
1415	* Clears an exception intercept in the specified VMCB.
1416	*
1417	* @param pVCpu The cross context virtual CPU structure.
1418	* @param pVmcb Pointer to the VM control block.
1419	* @param uXcpt The exception (X86_XCPT_*).
1420	*
1421	* @remarks This takes into account if we're executing a nested-guest and only
1422	* removes the exception intercept if both the guest -and- nested-guest
1423	* are not intercepting it.
1424	*/
1425	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPU pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1426	{
1427	Assert(uXcpt != X86_XCPT_DB);
1428	Assert(uXcpt != X86_XCPT_AC);
1429	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1430	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1431	{
1432	bool fRemove = true;
1433	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1434	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1435	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1436	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1437	{
1438	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1439	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1440	}
1441	# else
1442	RT_NOREF(pVCpu);
1443	# endif
1444	if (fRemove)
1445	{
1446	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1447	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1448	}
1449	}
1450	#else
1451	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1452	#endif
1453	}
1454
1455
1456	/**
1457	* Sets a control intercept in the specified VMCB.
1458	*
1459	* @param pVmcb Pointer to the VM control block.
1460	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1461	*/
1462	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1463	{
1464	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1465	{
1466	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1467	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1468	}
1469	}
1470
1471
1472	/**
1473	* Clears a control intercept in the specified VMCB.
1474	*
1475	* @returns @c true if the intercept is still set, @c false otherwise.
1476	* @param pVCpu The cross context virtual CPU structure.
1477	* @param pVmcb Pointer to the VM control block.
1478	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1479	*
1480	* @remarks This takes into account if we're executing a nested-guest and only
1481	* removes the control intercept if both the guest -and- nested-guest
1482	* are not intercepting it.
1483	*/
1484	static bool hmR0SvmClearCtrlIntercept(PVMCPU pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1485	{
1486	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1487	{
1488	bool fRemove = true;
1489	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1490	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1491	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1492	{
1493	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1494	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1495	}
1496	#else
1497	RT_NOREF(pVCpu);
1498	#endif
1499	if (fRemove)
1500	{
1501	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1502	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1503	}
1504	}
1505
1506	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1507	}
1508
1509
1510	/**
1511	* Exports the guest (or nested-guest) CR0 into the VMCB.
1512	*
1513	* @param pVCpu The cross context virtual CPU structure.
1514	* @param pVmcb Pointer to the VM control block.
1515	*
1516	* @remarks This assumes we always pre-load the guest FPU.
1517	* @remarks No-long-jump zone!!!
1518	*/
1519	static void hmR0SvmExportGuestCR0(PVMCPU pVCpu, PSVMVMCB pVmcb)
1520	{
1521	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1522
1523	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1524	uint64_t const uGuestCr0 = pCtx->cr0;
1525	uint64_t uShadowCr0 = uGuestCr0;
1526
1527	/* Always enable caching. */
1528	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1529
1530	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1531	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1532	{
1533	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1534	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1535	}
1536
1537	/*
1538	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1539	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1540	* see @bugref{7243#c103}.
1541	*/
1542	if (!(uGuestCr0 & X86_CR0_NE))
1543	{
1544	uShadowCr0 \|= X86_CR0_NE;
1545	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1546	}
1547	else
1548	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1549
1550	/*
1551	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1552	*
1553	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1554	* see @bugref{6944}.
1555	*
1556	* We also don't ever want to honor weird things like cache disable from the guest.
1557	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1558	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1559	*/
1560	if (uShadowCr0 == uGuestCr0)
1561	{
1562	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1563	{
1564	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1565	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1566	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1567	}
1568	else
1569	{
1570	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1571	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1572	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1573	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1574	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1575	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1576	}
1577	}
1578	else
1579	{
1580	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1581	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1582	}
1583	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1584
1585	Assert(!RT_HI_U32(uShadowCr0));
1586	if (pVmcb->guest.u64CR0 != uShadowCr0)
1587	{
1588	pVmcb->guest.u64CR0 = uShadowCr0;
1589	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1590	}
1591	}
1592
1593
1594	/**
1595	* Exports the guest (or nested-guest) CR3 into the VMCB.
1596	*
1597	* @param pVCpu The cross context virtual CPU structure.
1598	* @param pVmcb Pointer to the VM control block.
1599	*
1600	* @remarks No-long-jump zone!!!
1601	*/
1602	static void hmR0SvmExportGuestCR3(PVMCPU pVCpu, PSVMVMCB pVmcb)
1603	{
1604	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1605
1606	PVM pVM = pVCpu->CTX_SUFF(pVM);
1607	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1608	if (pVM->hm.s.fNestedPaging)
1609	{
1610	PGMMODE enmShwPagingMode;
1611	#if HC_ARCH_BITS == 32
1612	if (CPUMIsGuestInLongModeEx(pCtx))
1613	enmShwPagingMode = PGMMODE_AMD64_NX;
1614	else
1615	#endif
1616	enmShwPagingMode = PGMGetHostMode(pVM);
1617
1618	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
1619	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1620	pVmcb->guest.u64CR3 = pCtx->cr3;
1621	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1622	}
1623	else
1624	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1625
1626	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1627	}
1628
1629
1630	/**
1631	* Exports the guest (or nested-guest) CR4 into the VMCB.
1632	*
1633	* @param pVCpu The cross context virtual CPU structure.
1634	* @param pVmcb Pointer to the VM control block.
1635	*
1636	* @remarks No-long-jump zone!!!
1637	*/
1638	static int hmR0SvmExportGuestCR4(PVMCPU pVCpu, PSVMVMCB pVmcb)
1639	{
1640	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1641
1642	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1643	uint64_t uShadowCr4 = pCtx->cr4;
1644	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1645	{
1646	switch (pVCpu->hm.s.enmShadowMode)
1647	{
1648	case PGMMODE_REAL:
1649	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1650	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1651
1652	case PGMMODE_32_BIT: /* 32-bit paging. */
1653	uShadowCr4 &= ~X86_CR4_PAE;
1654	break;
1655
1656	case PGMMODE_PAE: /* PAE paging. */
1657	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1658	/** Must use PAE paging as we could use physical memory > 4 GB */
1659	uShadowCr4 \|= X86_CR4_PAE;
1660	break;
1661
1662	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1663	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1664	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1665	break;
1666	#else
1667	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1668	#endif
1669
1670	default: /* shut up gcc */
1671	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1672	}
1673	}
1674
1675	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1676	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1677
1678	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1679	if (uShadowCr4 == pCtx->cr4)
1680	{
1681	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1682	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1683	else
1684	{
1685	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1686	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1687	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1688	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1689	}
1690	}
1691	else
1692	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1693
1694	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking PGM mode changes. */
1695	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1696
1697	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1698	Assert(!RT_HI_U32(uShadowCr4));
1699	pVmcb->guest.u64CR4 = uShadowCr4;
1700	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1701
1702	return VINF_SUCCESS;
1703	}
1704
1705
1706	/**
1707	* Exports the guest (or nested-guest) control registers into the VMCB.
1708	*
1709	* @returns VBox status code.
1710	* @param pVCpu The cross context virtual CPU structure.
1711	* @param pVmcb Pointer to the VM control block.
1712	*
1713	* @remarks No-long-jump zone!!!
1714	*/
1715	static int hmR0SvmExportGuestControlRegs(PVMCPU pVCpu, PSVMVMCB pVmcb)
1716	{
1717	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1718
1719	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1720	{
1721	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1722	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1723
1724	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1725	{
1726	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1727	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1728	}
1729
1730	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1731	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1732
1733	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1734	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1735	{
1736	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1737	if (RT_FAILURE(rc))
1738	return rc;
1739	}
1740
1741	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1742	}
1743	return VINF_SUCCESS;
1744	}
1745
1746
1747	/**
1748	* Exports the guest (or nested-guest) segment registers into the VMCB.
1749	*
1750	* @returns VBox status code.
1751	* @param pVCpu The cross context virtual CPU structure.
1752	* @param pVmcb Pointer to the VM control block.
1753	*
1754	* @remarks No-long-jump zone!!!
1755	*/
1756	static void hmR0SvmExportGuestSegmentRegs(PVMCPU pVCpu, PSVMVMCB pVmcb)
1757	{
1758	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1759	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1760
1761	/* Guest segment registers. */
1762	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1763	{
1764	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1765	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1766
1767	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1768	{
1769	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1770	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1771	}
1772
1773	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1774	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1775
1776	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1777	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1778
1779	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1780	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1781
1782	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1783	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1784
1785	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1786	}
1787
1788	/* Guest TR. */
1789	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1790	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1791
1792	/* Guest LDTR. */
1793	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1794	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1795
1796	/* Guest GDTR. */
1797	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1798	{
1799	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1800	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1801	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1802	}
1803
1804	/* Guest IDTR. */
1805	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1806	{
1807	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1808	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1809	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1810	}
1811
1812	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1813	\| HM_CHANGED_GUEST_TABLE_MASK);
1814	}
1815
1816
1817	/**
1818	* Exports the guest (or nested-guest) MSRs into the VMCB.
1819	*
1820	* @param pVCpu The cross context virtual CPU structure.
1821	* @param pVmcb Pointer to the VM control block.
1822	*
1823	* @remarks No-long-jump zone!!!
1824	*/
1825	static void hmR0SvmExportGuestMsrs(PVMCPU pVCpu, PSVMVMCB pVmcb)
1826	{
1827	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1828	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1829
1830	/* Guest Sysenter MSRs. */
1831	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1832	{
1833	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1834	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1835
1836	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1837	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1838
1839	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1840	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1841	}
1842
1843	/*
1844	* Guest EFER MSR.
1845	* AMD-V requires guest EFER.SVME to be set. Weird.
1846	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1847	*/
1848	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1849	{
1850	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1851	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1852	}
1853
1854	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1855	if ( !CPUMIsGuestInLongModeEx(pCtx)
1856	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1857	{
1858	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1859	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1860	}
1861
1862	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1863	{
1864	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1865	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1866	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1867	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1868	}
1869
1870	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1871	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1872
1873	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1874	\| HM_CHANGED_GUEST_EFER_MSR
1875	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1876	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1877
1878	/*
1879	* Setup the PAT MSR (applicable for Nested Paging only).
1880	*
1881	* While guests can modify and see the modified values through the shadow values,
1882	* we shall not honor any guest modifications of this MSR to ensure caching is always
1883	* enabled similar to how we clear CR0.CD and NW bits.
1884	*
1885	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1886	*/
1887	pVmcb->guest.u64PAT = MSR_IA32_CR_PAT_INIT_VAL;
1888
1889	/* Enable the last branch record bit if LBR virtualization is enabled. */
1890	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1891	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1892	}
1893
1894
1895	/**
1896	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1897	* the necessary intercepts accordingly.
1898	*
1899	* @param pVCpu The cross context virtual CPU structure.
1900	* @param pVmcb Pointer to the VM control block.
1901	*
1902	* @remarks No-long-jump zone!!!
1903	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1904	*/
1905	static void hmR0SvmExportSharedDebugState(PVMCPU pVCpu, PSVMVMCB pVmcb)
1906	{
1907	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1908
1909	/*
1910	* Anyone single stepping on the host side? If so, we'll have to use the
1911	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1912	* the VMM level like the VT-x implementations does.
1913	*/
1914	bool fInterceptMovDRx = false;
1915	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1916	if (fStepping)
1917	{
1918	pVCpu->hm.s.fClearTrapFlag = true;
1919	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1920	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1921	}
1922
1923	if ( fStepping
1924	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1925	{
1926	/*
1927	* Use the combined guest and host DRx values found in the hypervisor
1928	* register set because the debugger has breakpoints active or someone
1929	* is single stepping on the host side.
1930	*
1931	* Note! DBGF expects a clean DR6 state before executing guest code.
1932	*/
1933	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1934	if ( CPUMIsGuestInLongModeEx(pCtx)
1935	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
1936	{
1937	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1938	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
1939	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
1940	}
1941	else
1942	#endif
1943	if (!CPUMIsHyperDebugStateActive(pVCpu))
1944	{
1945	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1946	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1947	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1948	}
1949
1950	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1951	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1952	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1953	{
1954	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1955	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1956	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1957	}
1958
1959	/** @todo If we cared, we could optimize to allow the guest to read registers
1960	* with the same values. */
1961	fInterceptMovDRx = true;
1962	pVCpu->hm.s.fUsingHyperDR7 = true;
1963	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1964	}
1965	else
1966	{
1967	/*
1968	* Update DR6, DR7 with the guest values if necessary.
1969	*/
1970	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1971	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1972	{
1973	pVmcb->guest.u64DR7 = pCtx->dr[7];
1974	pVmcb->guest.u64DR6 = pCtx->dr[6];
1975	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1976	}
1977	pVCpu->hm.s.fUsingHyperDR7 = false;
1978
1979	/*
1980	* If the guest has enabled debug registers, we need to load them prior to
1981	* executing guest code so they'll trigger at the right time.
1982	*/
1983	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1984	{
1985	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1986	if ( CPUMIsGuestInLongModeEx(pCtx)
1987	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
1988	{
1989	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1990	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1991	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
1992	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
1993	}
1994	else
1995	#endif
1996	if (!CPUMIsGuestDebugStateActive(pVCpu))
1997	{
1998	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1999	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2000	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2001	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2002	}
2003	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2004	}
2005	/*
2006	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2007	* intercept #DB as DR6 is updated in the VMCB.
2008	*
2009	* Note! If we cared and dared, we could skip intercepting \#DB here.
2010	* However, \#DB shouldn't be performance critical, so we'll play safe
2011	* and keep the code similar to the VT-x code and always intercept it.
2012	*/
2013	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
2014	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
2015	&& !CPUMIsGuestDebugStateActive(pVCpu))
2016	#else
2017	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2018	#endif
2019	{
2020	fInterceptMovDRx = true;
2021	}
2022	}
2023
2024	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2025	if (fInterceptMovDRx)
2026	{
2027	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2028	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2029	{
2030	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2031	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2032	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2033	}
2034	}
2035	else
2036	{
2037	if ( pVmcb->ctrl.u16InterceptRdDRx
2038	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2039	{
2040	pVmcb->ctrl.u16InterceptRdDRx = 0;
2041	pVmcb->ctrl.u16InterceptWrDRx = 0;
2042	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2043	}
2044	}
2045	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2046	}
2047
2048	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2049	/**
2050	* Exports the nested-guest hardware virtualization state into the nested-guest
2051	* VMCB.
2052	*
2053	* @param pVCpu The cross context virtual CPU structure.
2054	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2055	*
2056	* @remarks No-long-jump zone!!!
2057	*/
2058	static void hmR0SvmExportGuestHwvirtStateNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst)
2059	{
2060	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2061
2062	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2063	{
2064	/*
2065	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2066	* since SVM doesn't have a preemption timer.
2067	*
2068	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2069	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2070	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2071	*/
2072	PVM pVM = pVCpu->CTX_SUFF(pVM);
2073	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2074	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2075	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2076	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_PAUSE))
2077	{
2078	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2079	pVmcbNstGstCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2080	pVmcbNstGstCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2081	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2082	}
2083	else
2084	{
2085	pVmcbNstGstCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2086	pVmcbNstGstCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2087	}
2088
2089	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2090	}
2091	}
2092	#endif
2093
2094	/**
2095	* Exports the guest APIC TPR state into the VMCB.
2096	*
2097	* @returns VBox status code.
2098	* @param pVCpu The cross context virtual CPU structure.
2099	* @param pVmcb Pointer to the VM control block.
2100	*/
2101	static int hmR0SvmExportGuestApicTpr(PVMCPU pVCpu, PSVMVMCB pVmcb)
2102	{
2103	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2104	{
2105	PVM pVM = pVCpu->CTX_SUFF(pVM);
2106	if ( PDMHasApic(pVM)
2107	&& APICIsEnabled(pVCpu))
2108	{
2109	bool fPendingIntr;
2110	uint8_t u8Tpr;
2111	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2112	AssertRCReturn(rc, rc);
2113
2114	/* Assume that we need to trap all TPR accesses and thus need not check on
2115	every #VMEXIT if we should update the TPR. */
2116	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2117	pVCpu->hm.s.svm.fSyncVTpr = false;
2118
2119	if (!pVM->hm.s.fTPRPatchingActive)
2120	{
2121	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2122	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2123
2124	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2125	can deliver the interrupt to the guest. */
2126	if (fPendingIntr)
2127	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2128	else
2129	{
2130	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2131	pVCpu->hm.s.svm.fSyncVTpr = true;
2132	}
2133
2134	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2135	}
2136	else
2137	{
2138	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2139	pVmcb->guest.u64LSTAR = u8Tpr;
2140	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
2141
2142	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2143	if (fPendingIntr)
2144	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2145	else
2146	{
2147	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2148	pVCpu->hm.s.svm.fSyncVTpr = true;
2149	}
2150	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2151	}
2152	}
2153	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2154	}
2155	return VINF_SUCCESS;
2156	}
2157
2158
2159	/**
2160	* Sets up the exception interrupts required for guest (or nested-guest)
2161	* execution in the VMCB.
2162	*
2163	* @param pVCpu The cross context virtual CPU structure.
2164	* @param pVmcb Pointer to the VM control block.
2165	*
2166	* @remarks No-long-jump zone!!!
2167	*/
2168	static void hmR0SvmExportGuestXcptIntercepts(PVMCPU pVCpu, PSVMVMCB pVmcb)
2169	{
2170	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2171
2172	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2173	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_GUEST_XCPT_INTERCEPTS)
2174	{
2175	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2176	if (pVCpu->hm.s.fGIMTrapXcptUD)
2177	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2178	else
2179	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2180
2181	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2182	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2183	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2184	else
2185	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2186
2187	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2188	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_SVM_GUEST_XCPT_INTERCEPTS;
2189	}
2190	}
2191
2192
2193	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2194	/**
2195	* Merges guest and nested-guest intercepts for executing the nested-guest using
2196	* hardware-assisted SVM.
2197	*
2198	* This merges the guest and nested-guest intercepts in a way that if the outer
2199	* guest intercept is set we need to intercept it in the nested-guest as
2200	* well.
2201	*
2202	* @param pVCpu The cross context virtual CPU structure.
2203	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2204	*/
2205	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPU pVCpu)
2206	{
2207	PVM pVM = pVCpu->CTX_SUFF(pVM);
2208	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2209	PSVMVMCB pVmcbNstGst = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
2210	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2211
2212	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2213	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2214	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2215
2216	/* Always intercept CR4 writes for tracking PGM mode changes. */
2217	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2218
2219	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2220	if (!pVM->hm.s.fNestedPaging)
2221	{
2222	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2223	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2224	}
2225
2226	/** @todo Figure out debugging with nested-guests, till then just intercept
2227	* all DR[0-15] accesses. */
2228	pVmcbNstGstCtrl->u16InterceptRdDRx \|= 0xffff;
2229	pVmcbNstGstCtrl->u16InterceptWrDRx \|= 0xffff;
2230
2231	/*
2232	* Merge the guest's exception intercepts into the nested-guest VMCB.
2233	*
2234	* - \#UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2235	* while executing the nested-guest.
2236	*
2237	* - \#BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2238	* be tweaked later depending on how we wish to implement breakpoints.
2239	*
2240	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2241	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2242	*/
2243	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2244	pVmcbNstGstCtrl->u32InterceptXcpt \|= (pVmcb->ctrl.u32InterceptXcpt & ~( RT_BIT(X86_XCPT_UD)
2245	\| RT_BIT(X86_XCPT_BP)));
2246	#else
2247	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2248	#endif
2249
2250	/*
2251	* Adjust intercepts while executing the nested-guest that differ from the
2252	* outer guest intercepts.
2253	*
2254	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2255	* that belong to the nested-guest to the outer guest.
2256	*
2257	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2258	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2259	*/
2260	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2261	\| SVM_CTRL_INTERCEPT_VMMCALL))
2262	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2263
2264	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2265	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2266
2267	/* Finally, update the VMCB clean bits. */
2268	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2269	}
2270	#endif
2271
2272
2273	/**
2274	* Selects the appropriate function to run guest code.
2275	*
2276	* @returns VBox status code.
2277	* @param pVCpu The cross context virtual CPU structure.
2278	*
2279	* @remarks No-long-jump zone!!!
2280	*/
2281	static int hmR0SvmSelectVMRunHandler(PVMCPU pVCpu)
2282	{
2283	if (CPUMIsGuestInLongMode(pVCpu))
2284	{
2285	#ifndef VBOX_ENABLE_64_BITS_GUESTS
2286	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2287	#endif
2288	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
2289	#if HC_ARCH_BITS == 32
2290	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
2291	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
2292	#else
2293	/* 64-bit host or hybrid host. */
2294	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun64;
2295	#endif
2296	}
2297	else
2298	{
2299	/* Guest is not in long mode, use the 32-bit handler. */
2300	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
2301	}
2302	return VINF_SUCCESS;
2303	}
2304
2305
2306	/**
2307	* Enters the AMD-V session.
2308	*
2309	* @returns VBox status code.
2310	* @param pVCpu The cross context virtual CPU structure.
2311	* @param pHostCpu Pointer to the CPU info struct.
2312	*/
2313	VMMR0DECL(int) SVMR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
2314	{
2315	AssertPtr(pVCpu);
2316	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2317	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2318	RT_NOREF(pHostCpu);
2319
2320	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2321	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2322	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2323
2324	pVCpu->hm.s.fLeaveDone = false;
2325	return VINF_SUCCESS;
2326	}
2327
2328
2329	/**
2330	* Thread-context callback for AMD-V.
2331	*
2332	* @param enmEvent The thread-context event.
2333	* @param pVCpu The cross context virtual CPU structure.
2334	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2335	* @thread EMT(pVCpu)
2336	*/
2337	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
2338	{
2339	NOREF(fGlobalInit);
2340
2341	switch (enmEvent)
2342	{
2343	case RTTHREADCTXEVENT_OUT:
2344	{
2345	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2346	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2347	VMCPU_ASSERT_EMT(pVCpu);
2348
2349	/* No longjmps (log-flush, locks) in this fragile context. */
2350	VMMRZCallRing3Disable(pVCpu);
2351
2352	if (!pVCpu->hm.s.fLeaveDone)
2353	{
2354	hmR0SvmLeave(pVCpu, false /* fImportState */);
2355	pVCpu->hm.s.fLeaveDone = true;
2356	}
2357
2358	/* Leave HM context, takes care of local init (term). */
2359	int rc = HMR0LeaveCpu(pVCpu);
2360	AssertRC(rc); NOREF(rc);
2361
2362	/* Restore longjmp state. */
2363	VMMRZCallRing3Enable(pVCpu);
2364	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2365	break;
2366	}
2367
2368	case RTTHREADCTXEVENT_IN:
2369	{
2370	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2371	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2372	VMCPU_ASSERT_EMT(pVCpu);
2373
2374	/* No longjmps (log-flush, locks) in this fragile context. */
2375	VMMRZCallRing3Disable(pVCpu);
2376
2377	/*
2378	* Initialize the bare minimum state required for HM. This takes care of
2379	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2380	*/
2381	int rc = hmR0EnterCpu(pVCpu);
2382	AssertRC(rc); NOREF(rc);
2383	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2384	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2385
2386	pVCpu->hm.s.fLeaveDone = false;
2387
2388	/* Restore longjmp state. */
2389	VMMRZCallRing3Enable(pVCpu);
2390	break;
2391	}
2392
2393	default:
2394	break;
2395	}
2396	}
2397
2398
2399	/**
2400	* Saves the host state.
2401	*
2402	* @returns VBox status code.
2403	* @param pVCpu The cross context virtual CPU structure.
2404	*
2405	* @remarks No-long-jump zone!!!
2406	*/
2407	VMMR0DECL(int) SVMR0ExportHostState(PVMCPU pVCpu)
2408	{
2409	NOREF(pVCpu);
2410
2411	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2412	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2413	return VINF_SUCCESS;
2414	}
2415
2416
2417	/**
2418	* Exports the guest state from the guest-CPU context into the VMCB.
2419	*
2420	* The CPU state will be loaded from these fields on every successful VM-entry.
2421	* Also sets up the appropriate VMRUN function to execute guest code based on
2422	* the guest CPU mode.
2423	*
2424	* @returns VBox status code.
2425	* @param pVCpu The cross context virtual CPU structure.
2426	*
2427	* @remarks No-long-jump zone!!!
2428	*/
2429	static int hmR0SvmExportGuestState(PVMCPU pVCpu)
2430	{
2431	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2432
2433	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2434	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2435
2436	Assert(pVmcb);
2437	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
2438
2439	pVmcb->guest.u64RIP = pCtx->rip;
2440	pVmcb->guest.u64RSP = pCtx->rsp;
2441	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2442	pVmcb->guest.u64RAX = pCtx->rax;
2443	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2444	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2445	{
2446	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);
2447	pVmcb->ctrl.IntCtrl.n.u1VGif = pCtx->hwvirt.fGif;
2448	}
2449	#endif
2450
2451	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2452
2453	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2454	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2455
2456	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2457	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2458	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2459
2460	ASMSetFlags(fEFlags);
2461
2462	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2463	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2464	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2465
2466	rc = hmR0SvmSelectVMRunHandler(pVCpu);
2467	AssertRCReturn(rc, rc);
2468
2469	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2470	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( HM_CHANGED_GUEST_RIP
2471	\| HM_CHANGED_GUEST_RFLAGS
2472	\| HM_CHANGED_GUEST_GPRS_MASK
2473	\| HM_CHANGED_GUEST_X87
2474	\| HM_CHANGED_GUEST_SSE_AVX
2475	\| HM_CHANGED_GUEST_OTHER_XSAVE
2476	\| HM_CHANGED_GUEST_XCRx
2477	\| HM_CHANGED_GUEST_TSC_AUX
2478	\| HM_CHANGED_GUEST_OTHER_MSRS
2479	\| HM_CHANGED_GUEST_HWVIRT
2480	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_GUEST_XCPT_INTERCEPTS)));
2481
2482	#ifdef VBOX_STRICT
2483	/*
2484	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2485	* except for the host-context and/or shared host-guest context bits.
2486	*/
2487	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2488	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
2489	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2490	("fCtxChanged=%#RX64\n", fCtxChanged));
2491
2492	/*
2493	* If we need to log state that isn't always imported, we'll need to import them here.
2494	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2495	*/
2496	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2497	#endif
2498
2499	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2500	return VINF_SUCCESS;
2501	}
2502
2503
2504	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2505	/**
2506	* Merges the guest and nested-guest MSR permission bitmap.
2507	*
2508	* If the guest is intercepting an MSR we need to intercept it regardless of
2509	* whether the nested-guest is intercepting it or not.
2510	*
2511	* @param pHostCpu Pointer to the physical CPU HM info. struct.
2512	* @param pVCpu The cross context virtual CPU structure.
2513	*
2514	* @remarks No-long-jmp zone!!!
2515	*/
2516	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMGLOBALCPUINFO pHostCpu, PVMCPU pVCpu)
2517	{
2518	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hm.s.svm.pvMsrBitmap;
2519	uint64_t const pu64NstGstMsrpm = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
2520	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2521
2522	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2523	uint32_t const offRsvdQwords = 0x1800 >> 3;
2524	for (uint32_t i = 0; i < offRsvdQwords; i++)
2525	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2526	}
2527
2528
2529	/**
2530	* Caches the nested-guest VMCB fields before we modify them for execution using
2531	* hardware-assisted SVM.
2532	*
2533	* @returns true if the VMCB was previously already cached, false otherwise.
2534	* @param pVCpu The cross context virtual CPU structure.
2535	*
2536	* @sa HMSvmNstGstVmExitNotify.
2537	*/
2538	static bool hmR0SvmCacheVmcbNested(PVMCPU pVCpu)
2539	{
2540	/*
2541	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2542	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2543	*
2544	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2545	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2546	*/
2547	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2548	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2549	if (!fWasCached)
2550	{
2551	PCSVMVMCB pVmcbNstGst = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
2552	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2553	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2554	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2555	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2556	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2557	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2558	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2559	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2560	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2561	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2562	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2563	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2564	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2565	pVmcbNstGstCache->fCacheValid = true;
2566	Log4Func(("Cached VMCB fields\n"));
2567	}
2568
2569	return fWasCached;
2570	}
2571
2572
2573	/**
2574	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2575	*
2576	* This is done the first time we enter nested-guest execution using SVM R0
2577	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2578	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2579	*
2580	* @param pVCpu The cross context virtual CPU structure.
2581	*/
2582	static void hmR0SvmSetupVmcbNested(PVMCPU pVCpu)
2583	{
2584	PSVMVMCB pVmcbNstGst = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
2585	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2586
2587	/*
2588	* First cache the nested-guest VMCB fields we may potentially modify.
2589	*/
2590	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2591	if (!fVmcbCached)
2592	{
2593	/*
2594	* The IOPM of the nested-guest can be ignored because the the guest always
2595	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2596	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2597	*/
2598	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2599
2600	/*
2601	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2602	* nested-paging suddenly while executing a VM (see assertion at the end of
2603	* Trap0eHandler() in PGMAllBth.h).
2604	*/
2605	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
2606
2607	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2608	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2609
2610	/*
2611	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2612	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2613	*/
2614	pVCpu->hm.s.svm.fSyncVTpr = false;
2615
2616	#ifdef DEBUG_ramshankar
2617	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2618	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2619	#endif
2620
2621	/*
2622	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2623	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2624	*/
2625	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2626	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2627	\| SVM_CTRL_INTERCEPT_VMLOAD;
2628
2629	/*
2630	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2631	* CLGI/STGI instructions executed by the nested-guest.
2632	*/
2633	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2634	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2635	\| SVM_CTRL_INTERCEPT_STGI;
2636
2637	/* Merge the guest and nested-guest intercepts. */
2638	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2639
2640	/* Update the VMCB clean bits. */
2641	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2642	}
2643	else
2644	{
2645	Assert(!pVCpu->hm.s.svm.fSyncVTpr);
2646	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2647	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
2648	}
2649	}
2650
2651
2652	/**
2653	* Exports the nested-guest state into the VMCB.
2654	*
2655	* We need to export the entire state as we could be continuing nested-guest
2656	* execution at any point (not just immediately after VMRUN) and thus the VMCB
2657	* can be out-of-sync with the nested-guest state if it was executed in IEM.
2658	*
2659	* @returns VBox status code.
2660	* @param pVCpu The cross context virtual CPU structure.
2661	* @param pCtx Pointer to the guest-CPU context.
2662	*
2663	* @remarks No-long-jump zone!!!
2664	*/
2665	static int hmR0SvmExportGuestStateNested(PVMCPU pVCpu)
2666	{
2667	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2668
2669	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2670	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2671	Assert(pVmcbNstGst);
2672
2673	hmR0SvmSetupVmcbNested(pVCpu);
2674
2675	pVmcbNstGst->guest.u64RIP = pCtx->rip;
2676	pVmcbNstGst->guest.u64RSP = pCtx->rsp;
2677	pVmcbNstGst->guest.u64RFlags = pCtx->eflags.u32;
2678	pVmcbNstGst->guest.u64RAX = pCtx->rax;
2679
2680	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2681
2682	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcbNstGst);
2683	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2684
2685	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcbNstGst);
2686	hmR0SvmExportGuestMsrs(pVCpu, pVmcbNstGst);
2687	hmR0SvmExportGuestHwvirtStateNested(pVCpu, pVmcbNstGst);
2688
2689	ASMSetFlags(fEFlags);
2690
2691	/* Nested VGIF not supported yet. */
2692	Assert(!pVmcbNstGst->ctrl.IntCtrl.n.u1VGifEnable);
2693
2694	rc = hmR0SvmSelectVMRunHandler(pVCpu);
2695	AssertRCReturn(rc, rc);
2696
2697	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2698	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( HM_CHANGED_GUEST_RIP
2699	\| HM_CHANGED_GUEST_RFLAGS
2700	\| HM_CHANGED_GUEST_GPRS_MASK
2701	\| HM_CHANGED_GUEST_APIC_TPR
2702	\| HM_CHANGED_GUEST_X87
2703	\| HM_CHANGED_GUEST_SSE_AVX
2704	\| HM_CHANGED_GUEST_OTHER_XSAVE
2705	\| HM_CHANGED_GUEST_XCRx
2706	\| HM_CHANGED_GUEST_TSC_AUX
2707	\| HM_CHANGED_GUEST_OTHER_MSRS
2708	\| HM_CHANGED_SVM_GUEST_XCPT_INTERCEPTS
2709	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2710
2711	#ifdef VBOX_STRICT
2712	/*
2713	* All of the guest-CPU state and SVM keeper bits should be exported here by now, except
2714	* for the host-context and/or shared host-guest context bits.
2715	*/
2716	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2717	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
2718	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2719	("fCtxChanged=%#RX64\n", fCtxChanged));
2720
2721	/*
2722	* If we need to log state that isn't always imported, we'll need to import them here.
2723	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2724	*/
2725	hmR0SvmLogState(pVCpu, pVmcbNstGst, "hmR0SvmExportGuestStateNested", 0 /* fFlags /, 0 / uVerbose */);
2726	#endif
2727
2728	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2729	return rc;
2730	}
2731	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2732
2733
2734	/**
2735	* Exports the state shared between the host and guest (or nested-guest) into
2736	* the VMCB.
2737	*
2738	* @param pVCpu The cross context virtual CPU structure.
2739	* @param pVmcb Pointer to the VM control block.
2740	*
2741	* @remarks No-long-jump zone!!!
2742	*/
2743	static void hmR0SvmExportSharedState(PVMCPU pVCpu, PSVMVMCB pVmcb)
2744	{
2745	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2746	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2747
2748	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2749	{
2750	/** @todo Figure out stepping with nested-guest. */
2751	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2752	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
2753	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2754	else
2755	{
2756	pVmcb->guest.u64DR6 = pCtx->dr[6];
2757	pVmcb->guest.u64DR7 = pCtx->dr[7];
2758	}
2759	}
2760
2761	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2762	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2763	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2764	}
2765
2766
2767	/**
2768	* Worker for SVMR0ImportStateOnDemand.
2769	*
2770	* @param pVCpu The cross context virtual CPU structure.
2771	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2772	*/
2773	static void hmR0SvmImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
2774	{
2775	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2776
2777	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2778	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2779	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2780	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2781
2782	Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
2783
2784	/*
2785	* We disable interrupts to make the updating of the state and in particular
2786	* the fExtrn modification atomic wrt to preemption hooks.
2787	*/
2788	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2789
2790	fWhat &= pCtx->fExtrn;
2791	if (fWhat)
2792	{
2793	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2794	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2795	{
2796	if ( !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
2797	&& pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2798	{
2799	/* We don't yet support passing VGIF feature to the guest. */
2800	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fVGif);
2801	pCtx->hwvirt.fGif = pVmcbCtrl->IntCtrl.n.u1VGif;
2802	}
2803	}
2804
2805	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2806	{
2807	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2808	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2809	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2810	}
2811	#endif
2812
2813	if (fWhat & CPUMCTX_EXTRN_HM_SVM_INT_SHADOW)
2814	{
2815	if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2816	EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
2817	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2818	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2819	}
2820
2821	if (fWhat & CPUMCTX_EXTRN_RIP)
2822	pCtx->rip = pVmcbGuest->u64RIP;
2823
2824	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2825	pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
2826
2827	if (fWhat & CPUMCTX_EXTRN_RSP)
2828	pCtx->rsp = pVmcbGuest->u64RSP;
2829
2830	if (fWhat & CPUMCTX_EXTRN_RAX)
2831	pCtx->rax = pVmcbGuest->u64RAX;
2832
2833	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2834	{
2835	if (fWhat & CPUMCTX_EXTRN_CS)
2836	{
2837	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2838	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2839	/** @todo SELM might need to be fixed as it too should not care about the
2840	* granularity bit. See @bugref{6785}. */
2841	if ( !pCtx->cs.Attr.n.u1Granularity
2842	&& pCtx->cs.Attr.n.u1Present
2843	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2844	{
2845	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2846	pCtx->cs.Attr.n.u1Granularity = 1;
2847	}
2848	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2849	}
2850	if (fWhat & CPUMCTX_EXTRN_SS)
2851	{
2852	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2853	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2854	/*
2855	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2856	* VMCB and uses that and thus it's possible that when the CPL changes during
2857	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2858	* AMD Fusion CPUs with 64-bit guests.
2859	*
2860	* See AMD spec. 15.5.1 "Basic operation".
2861	*/
2862	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2863	uint8_t const uCpl = pVmcbGuest->u8CPL;
2864	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2865	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2866	}
2867	if (fWhat & CPUMCTX_EXTRN_DS)
2868	{
2869	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2870	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2871	}
2872	if (fWhat & CPUMCTX_EXTRN_ES)
2873	{
2874	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2875	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2876	}
2877	if (fWhat & CPUMCTX_EXTRN_FS)
2878	{
2879	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2880	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2881	}
2882	if (fWhat & CPUMCTX_EXTRN_GS)
2883	{
2884	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2885	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2886	}
2887	}
2888
2889	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2890	{
2891	if (fWhat & CPUMCTX_EXTRN_TR)
2892	{
2893	/*
2894	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2895	* are used between Intel and AMD, see @bugref{6208#c39}.
2896	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2897	*/
2898	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2899	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2900	{
2901	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2902	\|\| CPUMIsGuestInLongModeEx(pCtx))
2903	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2904	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2905	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2906	}
2907	}
2908
2909	if (fWhat & CPUMCTX_EXTRN_LDTR)
2910	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2911
2912	if (fWhat & CPUMCTX_EXTRN_GDTR)
2913	{
2914	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2915	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2916	}
2917
2918	if (fWhat & CPUMCTX_EXTRN_IDTR)
2919	{
2920	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2921	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2922	}
2923	}
2924
2925	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2926	{
2927	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2928	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2929	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2930	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2931	}
2932
2933	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2934	{
2935	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2936	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2937	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2938	}
2939
2940	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2941	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2942
2943	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2944	{
2945	if (fWhat & CPUMCTX_EXTRN_DR6)
2946	{
2947	if (!pVCpu->hm.s.fUsingHyperDR7)
2948	pCtx->dr[6] = pVmcbGuest->u64DR6;
2949	else
2950	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2951	}
2952
2953	if (fWhat & CPUMCTX_EXTRN_DR7)
2954	{
2955	if (!pVCpu->hm.s.fUsingHyperDR7)
2956	pCtx->dr[7] = pVmcbGuest->u64DR7;
2957	else
2958	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2959	}
2960	}
2961
2962	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2963	{
2964	if (fWhat & CPUMCTX_EXTRN_CR0)
2965	{
2966	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2967	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2968	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2969	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2970	CPUMSetGuestCR0(pVCpu, uCr0);
2971	VMMRZCallRing3Enable(pVCpu);
2972	}
2973
2974	if (fWhat & CPUMCTX_EXTRN_CR2)
2975	pCtx->cr2 = pVmcbGuest->u64CR2;
2976
2977	if (fWhat & CPUMCTX_EXTRN_CR3)
2978	{
2979	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2980	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2981	{
2982	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2983	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2984	}
2985	}
2986
2987	/* Changes to CR4 are always intercepted. */
2988	}
2989
2990	/* Update fExtrn. */
2991	pCtx->fExtrn &= ~fWhat;
2992
2993	/* If everything has been imported, clear the HM keeper bit. */
2994	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2995	{
2996	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2997	Assert(!pCtx->fExtrn);
2998	}
2999	}
3000	else
3001	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3002
3003	ASMSetFlags(fEFlags);
3004
3005	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
3006
3007	/*
3008	* Honor any pending CR3 updates.
3009	*
3010	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
3011	* -> hmR0SvmCallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
3012	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
3013	* handling -> hmR0SvmImportGuestState() and here we are.
3014	*
3015	* The reason for such complicated handling is because VM-exits that call into PGM expect
3016	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
3017	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
3018	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
3019	*
3020	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
3021	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
3022	*/
3023	if ( VMMRZCallRing3IsEnabled(pVCpu)
3024	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3025	{
3026	Assert(pCtx->cr3 == pVmcbGuest->u64CR3);
3027	PGMUpdateCR3(pVCpu, pCtx->cr3);
3028	}
3029	}
3030
3031
3032	/**
3033	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
3034	* context.
3035	*
3036	* Currently there is no residual state left in the CPU that is not updated in the
3037	* VMCB.
3038	*
3039	* @returns VBox status code.
3040	* @param pVCpu The cross context virtual CPU structure.
3041	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3042	*/
3043	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
3044	{
3045	hmR0SvmImportGuestState(pVCpu, fWhat);
3046	return VINF_SUCCESS;
3047	}
3048
3049
3050	/**
3051	* Does the necessary state syncing before returning to ring-3 for any reason
3052	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
3053	*
3054	* @param pVCpu The cross context virtual CPU structure.
3055	* @param fImportState Whether to import the guest state from the VMCB back
3056	* to the guest-CPU context.
3057	*
3058	* @remarks No-long-jmp zone!!!
3059	*/
3060	static void hmR0SvmLeave(PVMCPU pVCpu, bool fImportState)
3061	{
3062	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3063	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3064	Assert(VMMR0IsLogFlushDisabled(pVCpu));
3065
3066	/*
3067	* !!! IMPORTANT !!!
3068	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
3069	*/
3070
3071	/* Save the guest state if necessary. */
3072	if (fImportState)
3073	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3074
3075	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3076	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3077	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
3078
3079	/*
3080	* Restore host debug registers if necessary and resync on next R0 reentry.
3081	*/
3082	#ifdef VBOX_STRICT
3083	if (CPUMIsHyperDebugStateActive(pVCpu))
3084	{
3085	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb; /** @todo nested-guest. */
3086	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
3087	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
3088	}
3089	#endif
3090	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3091	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3092	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3093
3094	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3095	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3096	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3097	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3098	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3099	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3100
3101	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3102	}
3103
3104
3105	/**
3106	* Leaves the AMD-V session.
3107	*
3108	* Only used while returning to ring-3 either due to longjump or exits to
3109	* ring-3.
3110	*
3111	* @returns VBox status code.
3112	* @param pVCpu The cross context virtual CPU structure.
3113	*/
3114	static int hmR0SvmLeaveSession(PVMCPU pVCpu)
3115	{
3116	HM_DISABLE_PREEMPT(pVCpu);
3117	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3118	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3119
3120	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3121	and done this from the SVMR0ThreadCtxCallback(). */
3122	if (!pVCpu->hm.s.fLeaveDone)
3123	{
3124	hmR0SvmLeave(pVCpu, true /* fImportState */);
3125	pVCpu->hm.s.fLeaveDone = true;
3126	}
3127
3128	/*
3129	* !!! IMPORTANT !!!
3130	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
3131	*/
3132
3133	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3134	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3135	VMMR0ThreadCtxHookDisable(pVCpu);
3136
3137	/* Leave HM context. This takes care of local init (term). */
3138	int rc = HMR0LeaveCpu(pVCpu);
3139
3140	HM_RESTORE_PREEMPT();
3141	return rc;
3142	}
3143
3144
3145	/**
3146	* Does the necessary state syncing before doing a longjmp to ring-3.
3147	*
3148	* @returns VBox status code.
3149	* @param pVCpu The cross context virtual CPU structure.
3150	*
3151	* @remarks No-long-jmp zone!!!
3152	*/
3153	static int hmR0SvmLongJmpToRing3(PVMCPU pVCpu)
3154	{
3155	return hmR0SvmLeaveSession(pVCpu);
3156	}
3157
3158
3159	/**
3160	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3161	* any remaining host state) before we longjump to ring-3 and possibly get
3162	* preempted.
3163	*
3164	* @param pVCpu The cross context virtual CPU structure.
3165	* @param enmOperation The operation causing the ring-3 longjump.
3166	* @param pvUser The user argument, NULL (currently unused).
3167	*/
3168	static DECLCALLBACK(int) hmR0SvmCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
3169	{
3170	RT_NOREF_PV(pvUser);
3171
3172	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
3173	{
3174	/*
3175	* !!! IMPORTANT !!!
3176	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3177	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3178	*/
3179	VMMRZCallRing3RemoveNotification(pVCpu);
3180	VMMRZCallRing3Disable(pVCpu);
3181	HM_DISABLE_PREEMPT(pVCpu);
3182
3183	/* Import the entire guest state. */
3184	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3185
3186	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3187	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3188
3189	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3190	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3191
3192	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3193	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3194	VMMR0ThreadCtxHookDisable(pVCpu);
3195
3196	/* Leave HM context. This takes care of local init (term). */
3197	HMR0LeaveCpu(pVCpu);
3198
3199	HM_RESTORE_PREEMPT();
3200	return VINF_SUCCESS;
3201	}
3202
3203	Assert(pVCpu);
3204	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3205	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3206
3207	VMMRZCallRing3Disable(pVCpu);
3208	Assert(VMMR0IsLogFlushDisabled(pVCpu));
3209
3210	Log4Func(("Calling hmR0SvmLongJmpToRing3\n"));
3211	int rc = hmR0SvmLongJmpToRing3(pVCpu);
3212	AssertRCReturn(rc, rc);
3213
3214	VMMRZCallRing3Enable(pVCpu);
3215	return VINF_SUCCESS;
3216	}
3217
3218
3219	/**
3220	* Take necessary actions before going back to ring-3.
3221	*
3222	* An action requires us to go back to ring-3. This function does the necessary
3223	* steps before we can safely return to ring-3. This is not the same as longjmps
3224	* to ring-3, this is voluntary.
3225	*
3226	* @returns VBox status code.
3227	* @param pVCpu The cross context virtual CPU structure.
3228	* @param rcExit The reason for exiting to ring-3. Can be
3229	* VINF_VMM_UNKNOWN_RING3_CALL.
3230	*/
3231	static int hmR0SvmExitToRing3(PVMCPU pVCpu, int rcExit)
3232	{
3233	Assert(pVCpu);
3234	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3235
3236	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3237	VMMRZCallRing3Disable(pVCpu);
3238	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", rcExit, (uint64_t)pVCpu->fLocalForcedActions,
3239	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3240
3241	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3242	if (pVCpu->hm.s.Event.fPending)
3243	{
3244	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3245	Assert(!pVCpu->hm.s.Event.fPending);
3246	}
3247
3248	/* Sync. the necessary state for going back to ring-3. */
3249	hmR0SvmLeaveSession(pVCpu);
3250	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3251
3252	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3253	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3254	\| CPUM_CHANGED_LDTR
3255	\| CPUM_CHANGED_GDTR
3256	\| CPUM_CHANGED_IDTR
3257	\| CPUM_CHANGED_TR
3258	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3259	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
3260	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3261	{
3262	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3263	}
3264
3265	/* Update the exit-to-ring 3 reason. */
3266	pVCpu->hm.s.rcLastExitToR3 = rcExit;
3267
3268	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3269	if ( rcExit != VINF_EM_RAW_INTERRUPT
3270	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3271	{
3272	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3273	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3274	}
3275
3276	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3277
3278	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
3279	VMMRZCallRing3RemoveNotification(pVCpu);
3280	VMMRZCallRing3Enable(pVCpu);
3281
3282	/*
3283	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3284	* and if we're injecting an event we should have a TRPM trap pending.
3285	*/
3286	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3287	pVCpu->hm.s.u32HMError = rcExit,
3288	VERR_SVM_IPE_5);
3289	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3290	pVCpu->hm.s.u32HMError = rcExit,
3291	VERR_SVM_IPE_4);
3292
3293	return rcExit;
3294	}
3295
3296
3297	/**
3298	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3299	* intercepts.
3300	*
3301	* @param pVCpu The cross context virtual CPU structure.
3302	* @param pVmcb Pointer to the VM control block.
3303	*
3304	* @remarks No-long-jump zone!!!
3305	*/
3306	static void hmR0SvmUpdateTscOffsetting(PVMCPU pVCpu, PSVMVMCB pVmcb)
3307	{
3308	/*
3309	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3310	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3311	* RDTSC/RDTSCP as well.
3312	*/
3313	bool fParavirtTsc;
3314	uint64_t uTscOffset;
3315	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3316
3317	bool fIntercept;
3318	if (fCanUseRealTsc)
3319	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3320	else
3321	{
3322	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3323	fIntercept = true;
3324	}
3325
3326	if (!fIntercept)
3327	{
3328	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3329	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3330	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3331	uTscOffset = HMSvmNstGstApplyTscOffset(pVCpu, uTscOffset);
3332	#endif
3333
3334	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3335	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3336	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3337
3338	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
3339	}
3340	else
3341	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
3342
3343	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3344	information before every VM-entry, hence we have nothing to do here at the moment. */
3345	if (fParavirtTsc)
3346	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3347	}
3348
3349
3350	/**
3351	* Sets an event as a pending event to be injected into the guest.
3352	*
3353	* @param pVCpu The cross context virtual CPU structure.
3354	* @param pEvent Pointer to the SVM event.
3355	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3356	* page-fault.
3357	*
3358	* @remarks Statistics counter assumes this is a guest event being reflected to
3359	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3360	*/
3361	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPU pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3362	{
3363	Assert(!pVCpu->hm.s.Event.fPending);
3364	Assert(pEvent->n.u1Valid);
3365
3366	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3367	pVCpu->hm.s.Event.fPending = true;
3368	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3369
3370	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3371	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3372	}
3373
3374
3375	/**
3376	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3377	*
3378	* @param pVCpu The cross context virtual CPU structure.
3379	*/
3380	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPU pVCpu)
3381	{
3382	SVMEVENT Event;
3383	Event.u = 0;
3384	Event.n.u1Valid = 1;
3385	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3386	Event.n.u8Vector = X86_XCPT_UD;
3387	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3388	}
3389
3390
3391	/**
3392	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3393	*
3394	* @param pVCpu The cross context virtual CPU structure.
3395	*/
3396	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPU pVCpu)
3397	{
3398	SVMEVENT Event;
3399	Event.u = 0;
3400	Event.n.u1Valid = 1;
3401	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3402	Event.n.u8Vector = X86_XCPT_DB;
3403	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3404	}
3405
3406
3407	/**
3408	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3409	*
3410	* @param pVCpu The cross context virtual CPU structure.
3411	* @param u32ErrCode The error-code for the page-fault.
3412	* @param uFaultAddress The page fault address (CR2).
3413	*
3414	* @remarks This updates the guest CR2 with @a uFaultAddress!
3415	*/
3416	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPU pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3417	{
3418	SVMEVENT Event;
3419	Event.u = 0;
3420	Event.n.u1Valid = 1;
3421	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3422	Event.n.u8Vector = X86_XCPT_PF;
3423	Event.n.u1ErrorCodeValid = 1;
3424	Event.n.u32ErrorCode = u32ErrCode;
3425
3426	/* Update CR2 of the guest. */
3427	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3428	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3429	{
3430	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3431	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3432	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3433	}
3434
3435	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3436	}
3437
3438
3439	/**
3440	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3441	*
3442	* @param pVCpu The cross context virtual CPU structure.
3443	*/
3444	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPU pVCpu)
3445	{
3446	SVMEVENT Event;
3447	Event.u = 0;
3448	Event.n.u1Valid = 1;
3449	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3450	Event.n.u8Vector = X86_XCPT_MF;
3451	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3452	}
3453
3454
3455	/**
3456	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3457	*
3458	* @param pVCpu The cross context virtual CPU structure.
3459	*/
3460	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPU pVCpu)
3461	{
3462	SVMEVENT Event;
3463	Event.u = 0;
3464	Event.n.u1Valid = 1;
3465	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3466	Event.n.u8Vector = X86_XCPT_DF;
3467	Event.n.u1ErrorCodeValid = 1;
3468	Event.n.u32ErrorCode = 0;
3469	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3470	}
3471
3472
3473	/**
3474	* Injects an event into the guest upon VMRUN by updating the relevant field
3475	* in the VMCB.
3476	*
3477	* @param pVCpu The cross context virtual CPU structure.
3478	* @param pVmcb Pointer to the guest VM control block.
3479	* @param pEvent Pointer to the event.
3480	*
3481	* @remarks No-long-jump zone!!!
3482	* @remarks Requires CR0!
3483	*/
3484	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPU pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3485	{
3486	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3487	pVmcb->ctrl.EventInject.u = pEvent->u;
3488	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3489	RT_NOREF(pVCpu);
3490
3491	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3492	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3493	}
3494
3495
3496
3497	/**
3498	* Converts any TRPM trap into a pending HM event. This is typically used when
3499	* entering from ring-3 (not longjmp returns).
3500	*
3501	* @param pVCpu The cross context virtual CPU structure.
3502	*/
3503	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPU pVCpu)
3504	{
3505	Assert(TRPMHasTrap(pVCpu));
3506	Assert(!pVCpu->hm.s.Event.fPending);
3507
3508	uint8_t uVector;
3509	TRPMEVENT enmTrpmEvent;
3510	RTGCUINT uErrCode;
3511	RTGCUINTPTR GCPtrFaultAddress;
3512	uint8_t cbInstr;
3513
3514	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
3515	AssertRC(rc);
3516
3517	SVMEVENT Event;
3518	Event.u = 0;
3519	Event.n.u1Valid = 1;
3520	Event.n.u8Vector = uVector;
3521
3522	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3523	if (enmTrpmEvent == TRPM_TRAP)
3524	{
3525	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3526	switch (uVector)
3527	{
3528	case X86_XCPT_NMI:
3529	{
3530	Event.n.u3Type = SVM_EVENT_NMI;
3531	break;
3532	}
3533
3534	case X86_XCPT_PF:
3535	case X86_XCPT_DF:
3536	case X86_XCPT_TS:
3537	case X86_XCPT_NP:
3538	case X86_XCPT_SS:
3539	case X86_XCPT_GP:
3540	case X86_XCPT_AC:
3541	{
3542	Event.n.u1ErrorCodeValid = 1;
3543	Event.n.u32ErrorCode = uErrCode;
3544	break;
3545	}
3546	}
3547	}
3548	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3549	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3550	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3551	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3552	else
3553	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3554
3555	rc = TRPMResetTrap(pVCpu);
3556	AssertRC(rc);
3557
3558	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3559	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3560
3561	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3562	}
3563
3564
3565	/**
3566	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3567	* AMD-V to execute any instruction.
3568	*
3569	* @param pVCpu The cross context virtual CPU structure.
3570	*/
3571	static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu)
3572	{
3573	Assert(pVCpu->hm.s.Event.fPending);
3574	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3575
3576	SVMEVENT Event;
3577	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3578
3579	uint8_t uVector = Event.n.u8Vector;
3580	uint8_t uVectorType = Event.n.u3Type;
3581	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event);
3582
3583	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, uVectorType));
3584
3585	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3586	AssertRC(rc);
3587
3588	if (Event.n.u1ErrorCodeValid)
3589	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3590
3591	if ( uVectorType == SVM_EVENT_EXCEPTION
3592	&& uVector == X86_XCPT_PF)
3593	{
3594	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3595	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3596	}
3597	else if (uVectorType == SVM_EVENT_SOFTWARE_INT)
3598	{
3599	AssertMsg( uVectorType == SVM_EVENT_SOFTWARE_INT
3600	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
3601	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
3602	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3603	}
3604	pVCpu->hm.s.Event.fPending = false;
3605	}
3606
3607
3608	/**
3609	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3610	* now.
3611	*
3612	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3613	* @param pVCpu The cross context virtual CPU structure.
3614	*
3615	* @remarks No-long-jump zone!!!
3616	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3617	*/
3618	static bool hmR0SvmIsIntrShadowActive(PVMCPU pVCpu)
3619	{
3620	/*
3621	* Instructions like STI and MOV SS inhibit interrupts till the next instruction
3622	* completes. Check if we should inhibit interrupts or clear any existing
3623	* interrupt inhibition.
3624	*/
3625	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3626	{
3627	if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
3628	{
3629	/*
3630	* We can clear the inhibit force flag as even if we go back to the recompiler
3631	* without executing guest code in AMD-V, the flag's condition to be cleared is
3632	* met and thus the cleared state is correct.
3633	*/
3634	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3635	return false;
3636	}
3637	return true;
3638	}
3639	return false;
3640	}
3641
3642
3643	/**
3644	* Sets the virtual interrupt intercept control in the VMCB.
3645	*
3646	* @param pVCpu The cross context virtual CPU structure.
3647	* @param pVmcb Pointer to the VM control block.
3648	*/
3649	static void hmR0SvmSetIntWindowExiting(PVMCPU pVCpu, PSVMVMCB pVmcb)
3650	{
3651	/*
3652	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3653	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3654	* (updating ISR at the right time) and inject the interrupt.
3655	*
3656	* With AVIC is supported, we could make use of the asynchronously delivery without
3657	* #VMEXIT and we would be passing the AVIC page to SVM.
3658	*
3659	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3660	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3661	*/
3662	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3663	/*
3664	* Currently we don't overlay interupt windows and if there's any V_IRQ pending in the
3665	* nested-guest VMCB, we avoid setting up any interrupt window on behalf of the outer
3666	* guest.
3667	*/
3668	/** @todo Does this mean we end up prioritizing virtual interrupt
3669	* delivery/window over a physical interrupt (from the outer guest)
3670	* might be pending? */
3671	bool const fEnableIntWindow = !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
3672	if (!fEnableIntWindow)
3673	{
3674	Assert(CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx));
3675	Log4(("Nested-guest V_IRQ already pending\n"));
3676	}
3677	#else
3678	bool const fEnableIntWindow = true;
3679	RT_NOREF(pVCpu);
3680	#endif
3681	if (fEnableIntWindow)
3682	{
3683	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3684	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3685	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3686	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3687	Log4(("Set VINTR intercept\n"));
3688	}
3689	}
3690
3691
3692	/**
3693	* Clears the virtual interrupt intercept control in the VMCB as
3694	* we are figured the guest is unable process any interrupts
3695	* at this point of time.
3696	*
3697	* @param pVCpu The cross context virtual CPU structure.
3698	* @param pVmcb Pointer to the VM control block.
3699	*/
3700	static void hmR0SvmClearIntWindowExiting(PVMCPU pVCpu, PSVMVMCB pVmcb)
3701	{
3702	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3703	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3704	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3705	{
3706	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3707	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3708	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3709	Log4(("Cleared VINTR intercept\n"));
3710	}
3711	}
3712
3713	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3714	/**
3715	* Evaluates the event to be delivered to the nested-guest and sets it as the
3716	* pending event.
3717	*
3718	* @returns VBox strict status code.
3719	* @param pVCpu The cross context virtual CPU structure.
3720	*/
3721	static VBOXSTRICTRC hmR0SvmEvaluatePendingEventNested(PVMCPU pVCpu)
3722	{
3723	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3724	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
3725	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3726	\| CPUMCTX_EXTRN_RFLAGS
3727	\| CPUMCTX_EXTRN_HM_SVM_INT_SHADOW
3728	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3729
3730	Assert(!pVCpu->hm.s.Event.fPending);
3731	Assert(pCtx->hwvirt.fGif);
3732	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3733	Assert(pVmcb);
3734
3735	bool const fVirtualGif = CPUMGetSvmNstGstVGif(pCtx);
3736	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3737	bool const fBlockNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3738
3739	Log4Func(("fVirtualGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3740	fVirtualGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3741	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3742
3743	/** @todo SMI. SMIs take priority over NMIs. */
3744
3745	/*
3746	* Check if the guest can receive NMIs.
3747	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3748	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3749	*/
3750	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3751	&& !fBlockNmi)
3752	{
3753	if ( fVirtualGif
3754	&& !fIntShadow)
3755	{
3756	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3757	{
3758	Log4(("Intercepting NMI -> #VMEXIT\n"));
3759	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3760	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3761	}
3762
3763	Log4(("Setting NMI pending for injection\n"));
3764	SVMEVENT Event;
3765	Event.u = 0;
3766	Event.n.u1Valid = 1;
3767	Event.n.u8Vector = X86_XCPT_NMI;
3768	Event.n.u3Type = SVM_EVENT_NMI;
3769	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3770	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3771	}
3772	else if (!fVirtualGif)
3773	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3774	else
3775	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3776	}
3777	/*
3778	* Check if the nested-guest can receive external interrupts (generated by the guest's
3779	* PIC/APIC).
3780	*
3781	* External intercepts, NMI, SMI etc. from the physical CPU are -always- intercepted
3782	* when executing using hardware-assisted SVM, see HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS.
3783	*
3784	* External interrupts that are generated for the outer guest may be intercepted
3785	* depending on how the nested-guest VMCB was programmed by guest software.
3786	*
3787	* Physical interrupts always take priority over virtual interrupts,
3788	* see AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3789	*/
3790	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3791	&& !pVCpu->hm.s.fSingleInstruction)
3792	{
3793	if ( fVirtualGif
3794	&& !fIntShadow
3795	&& CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx))
3796	{
3797	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3798	{
3799	Log4(("Intercepting INTR -> #VMEXIT\n"));
3800	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3801	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3802	}
3803
3804	uint8_t u8Interrupt;
3805	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3806	if (RT_SUCCESS(rc))
3807	{
3808	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3809	SVMEVENT Event;
3810	Event.u = 0;
3811	Event.n.u1Valid = 1;
3812	Event.n.u8Vector = u8Interrupt;
3813	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3814	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3815	}
3816	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3817	{
3818	/*
3819	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3820	* updated eventually when the TPR is written by the guest.
3821	*/
3822	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3823	}
3824	else
3825	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3826	}
3827	else if (!fVirtualGif)
3828	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3829	else
3830	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3831	}
3832
3833	return VINF_SUCCESS;
3834	}
3835	#endif
3836
3837	/**
3838	* Evaluates the event to be delivered to the guest and sets it as the pending
3839	* event.
3840	*
3841	* @param pVCpu The cross context virtual CPU structure.
3842	*/
3843	static void hmR0SvmEvaluatePendingEvent(PVMCPU pVCpu)
3844	{
3845	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3846	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3847	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3848	\| CPUMCTX_EXTRN_RFLAGS
3849	\| CPUMCTX_EXTRN_HM_SVM_INT_SHADOW);
3850
3851	Assert(!pVCpu->hm.s.Event.fPending);
3852	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3853	Assert(pVmcb);
3854
3855	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3856	bool const fGif = pCtx->hwvirt.fGif;
3857	#else
3858	bool const fGif = true;
3859	#endif
3860	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3861	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
3862	bool const fBlockNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3863
3864	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fBlockInt=%RTbool fIntShadow=%RTbool fIntPending=%RTbool NMI pending=%RTbool\n",
3865	fGif, fBlockNmi, fBlockInt, fIntShadow,
3866	VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3867	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3868
3869	/** @todo SMI. SMIs take priority over NMIs. */
3870
3871	/*
3872	* Check if the guest can receive NMIs.
3873	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3874	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3875	*/
3876	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3877	&& !fBlockNmi)
3878	{
3879	if ( fGif
3880	&& !fIntShadow)
3881	{
3882	Log4(("Setting NMI pending for injection\n"));
3883	SVMEVENT Event;
3884	Event.u = 0;
3885	Event.n.u1Valid = 1;
3886	Event.n.u8Vector = X86_XCPT_NMI;
3887	Event.n.u3Type = SVM_EVENT_NMI;
3888	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3889	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3890	}
3891	else if (!fGif)
3892	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3893	else
3894	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3895	}
3896	/*
3897	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3898	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3899	* it from the APIC device.
3900	*/
3901	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3902	&& !pVCpu->hm.s.fSingleInstruction)
3903	{
3904	if ( fGif
3905	&& !fBlockInt
3906	&& !fIntShadow)
3907	{
3908	uint8_t u8Interrupt;
3909	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3910	if (RT_SUCCESS(rc))
3911	{
3912	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3913	SVMEVENT Event;
3914	Event.u = 0;
3915	Event.n.u1Valid = 1;
3916	Event.n.u8Vector = u8Interrupt;
3917	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3918	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3919	}
3920	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3921	{
3922	/*
3923	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3924	* updated eventually when the TPR is written by the guest.
3925	*/
3926	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3927	}
3928	else
3929	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3930	}
3931	else if (!fGif)
3932	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3933	else
3934	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3935	}
3936	}
3937
3938
3939	/**
3940	* Injects any pending events into the guest (or nested-guest).
3941	*
3942	* @param pVCpu The cross context virtual CPU structure.
3943	* @param pVmcb Pointer to the VM control block.
3944	*
3945	* @remarks Must only be called when we are guaranteed to enter
3946	* hardware-assisted SVM execution and not return to ring-3
3947	* prematurely.
3948	*/
3949	static void hmR0SvmInjectPendingEvent(PVMCPU pVCpu, PSVMVMCB pVmcb)
3950	{
3951	Assert(!TRPMHasTrap(pVCpu));
3952	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3953
3954	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3955	#ifdef VBOX_STRICT
3956	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3957	bool const fGif = pCtx->hwvirt.fGif;
3958	bool fAllowInt = fGif;
3959	if (fGif)
3960	{
3961	/*
3962	* For nested-guests we have no way to determine if we're injecting a physical or
3963	* virtual interrupt at this point. Hence the partial verification below.
3964	*/
3965	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3966	fAllowInt = CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx) \|\| CPUMCanSvmNstGstTakeVirtIntr(pVCpu, pCtx);
3967	else
3968	fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
3969	}
3970	#endif
3971
3972	if (pVCpu->hm.s.Event.fPending)
3973	{
3974	SVMEVENT Event;
3975	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3976	Assert(Event.n.u1Valid);
3977
3978	/*
3979	* Validate event injection pre-conditions.
3980	*/
3981	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3982	{
3983	Assert(fAllowInt);
3984	Assert(!fIntShadow);
3985	}
3986	else if (Event.n.u3Type == SVM_EVENT_NMI)
3987	{
3988	Assert(fGif);
3989	Assert(!fIntShadow);
3990	}
3991
3992	/*
3993	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3994	* do this only when we are surely going to inject the NMI as otherwise if we return
3995	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3996	* SVM R0.
3997	*
3998	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3999	* which will set the VMCS field after actually delivering the NMI which we read on
4000	* VM-exit to determine the state.
4001	*/
4002	if ( Event.n.u3Type == SVM_EVENT_NMI
4003	&& Event.n.u8Vector == X86_XCPT_NMI
4004	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
4005	{
4006	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
4007	}
4008
4009	/*
4010	* Inject it (update VMCB for injection by the hardware).
4011	*/
4012	Log4(("Injecting pending HM event\n"));
4013	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
4014	pVCpu->hm.s.Event.fPending = false;
4015
4016	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
4017	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
4018	else
4019	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
4020	}
4021	else
4022	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
4023
4024	/*
4025	* We could have injected an NMI through IEM and continue guest execution using
4026	* hardware-assisted SVM. In which case, we would not have any events pending (above)
4027	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
4028	*/
4029	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
4030	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
4031
4032	/*
4033	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
4034	*
4035	* For nested-guests: We need to update it too for the scenario where IEM executes
4036	* the nested-guest but execution later continues here with an interrupt shadow active.
4037	*/
4038	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
4039	}
4040
4041
4042	/**
4043	* Reports world-switch error and dumps some useful debug info.
4044	*
4045	* @param pVCpu The cross context virtual CPU structure.
4046	* @param rcVMRun The return code from VMRUN (or
4047	* VERR_SVM_INVALID_GUEST_STATE for invalid
4048	* guest-state).
4049	*/
4050	static void hmR0SvmReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun)
4051	{
4052	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4053	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4054	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4055
4056	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
4057	{
4058	#ifdef VBOX_STRICT
4059	hmR0DumpRegs(pVCpu);
4060	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
4061	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
4062	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
4063	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
4064	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
4065	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
4066	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
4067	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
4068	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
4069	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
4070	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
4071
4072	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
4073	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
4074	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
4075
4076	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
4077	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
4078	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
4079	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
4080	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
4081	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
4082	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
4083	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
4084	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
4085	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
4086	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
4087	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
4088
4089	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
4090	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
4091	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
4092	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
4093	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
4094	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
4095	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
4096	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
4097	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
4098	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
4099	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
4100	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
4101	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
4102	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
4103	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
4104	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
4105	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
4106	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
4107	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
4108	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
4109
4110	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
4111
4112	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
4113	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
4114
4115	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
4116	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
4117	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
4118	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
4119	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
4120	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
4121	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
4122	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
4123	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
4124	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
4125	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
4126	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
4127	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
4128	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
4129	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
4130	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
4131	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
4132	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
4133	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
4134	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
4135
4136	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
4137	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
4138
4139	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
4140	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
4141	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
4142	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
4143
4144	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
4145	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
4146
4147	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
4148	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
4149	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
4150	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
4151
4152	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
4153	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
4154	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
4155	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
4156	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
4157	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
4158	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
4159
4160	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
4161	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
4162	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
4163	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
4164
4165	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
4166	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
4167	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
4168
4169	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
4170	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
4171	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
4172	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
4173	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
4174	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
4175	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
4176	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
4177	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
4178	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
4179	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
4180	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
4181
4182	NOREF(pVmcb);
4183	#endif /* VBOX_STRICT */
4184	}
4185	else
4186	Log4Func(("rcVMRun=%d\n", rcVMRun));
4187	}
4188
4189
4190	/**
4191	* Check per-VM and per-VCPU force flag actions that require us to go back to
4192	* ring-3 for one reason or another.
4193	*
4194	* @returns VBox status code (information status code included).
4195	* @retval VINF_SUCCESS if we don't have any actions that require going back to
4196	* ring-3.
4197	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
4198	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
4199	* interrupts)
4200	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
4201	* all EMTs to be in ring-3.
4202	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
4203	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
4204	* to the EM loop.
4205	*
4206	* @param pVCpu The cross context virtual CPU structure.
4207	*/
4208	static int hmR0SvmCheckForceFlags(PVMCPU pVCpu)
4209	{
4210	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4211	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
4212
4213	/* Could happen as a result of longjump. */
4214	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
4215	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
4216
4217	/* Update pending interrupts into the APIC's IRR. */
4218	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
4219	APICUpdatePendingInterrupts(pVCpu);
4220
4221	PVM pVM = pVCpu->CTX_SUFF(pVM);
4222	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
4223	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
4224	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
4225	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
4226	{
4227	/* Pending PGM C3 sync. */
4228	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
4229	{
4230	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
4231	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4232	if (rc != VINF_SUCCESS)
4233	{
4234	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
4235	return rc;
4236	}
4237	}
4238
4239	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4240	/* -XXX- what was that about single stepping? */
4241	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
4242	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4243	{
4244	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4245	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
4246	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
4247	return rc;
4248	}
4249
4250	/* Pending VM request packets, such as hardware interrupts. */
4251	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4252	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4253	{
4254	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4255	return VINF_EM_PENDING_REQUEST;
4256	}
4257
4258	/* Pending PGM pool flushes. */
4259	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4260	{
4261	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4262	return VINF_PGM_POOL_FLUSH_PENDING;
4263	}
4264
4265	/* Pending DMA requests. */
4266	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4267	{
4268	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4269	return VINF_EM_RAW_TO_R3;
4270	}
4271	}
4272
4273	return VINF_SUCCESS;
4274	}
4275
4276
4277	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4278	/**
4279	* Does the preparations before executing nested-guest code in AMD-V.
4280	*
4281	* @returns VBox status code (informational status codes included).
4282	* @retval VINF_SUCCESS if we can proceed with running the guest.
4283	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4284	*
4285	* @param pVCpu The cross context virtual CPU structure.
4286	* @param pSvmTransient Pointer to the SVM transient structure.
4287	*
4288	* @remarks Same caveats regarding longjumps as hmR0SvmPreRunGuest applies.
4289	* @sa hmR0SvmPreRunGuest.
4290	*/
4291	static int hmR0SvmPreRunGuestNested(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
4292	{
4293	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4294	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4295	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4296
4297	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM_ONLY_IN_IEM
4298	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4299	return VINF_EM_RESCHEDULE_REM;
4300	#endif
4301
4302	/* Check force flag actions that might require us to go back to ring-3. */
4303	int rc = hmR0SvmCheckForceFlags(pVCpu);
4304	if (rc != VINF_SUCCESS)
4305	return rc;
4306
4307	if (TRPMHasTrap(pVCpu))
4308	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4309	else if (!pVCpu->hm.s.Event.fPending)
4310	{
4311	VBOXSTRICTRC rcStrict = hmR0SvmEvaluatePendingEventNested(pVCpu);
4312	if ( rcStrict != VINF_SUCCESS
4313	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4314	return VBOXSTRICTRC_VAL(rcStrict);
4315	}
4316
4317	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4318
4319	/*
4320	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4321	* Just do it in software, see @bugref{8411}.
4322	* NB: If we could continue a task switch exit we wouldn't need to do this.
4323	*/
4324	PVM pVM = pVCpu->CTX_SUFF(pVM);
4325	if (RT_UNLIKELY( !pVM->hm.s.svm.u32Features
4326	&& pVCpu->hm.s.Event.fPending
4327	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4328	{
4329	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4330	}
4331
4332	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4333	Assert(!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4334	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4335	#endif
4336
4337	/*
4338	* Export the nested-guest state bits that are not shared with the host in any way as we
4339	* can longjmp or get preempted in the midst of exporting some of the state.
4340	*/
4341	rc = hmR0SvmExportGuestStateNested(pVCpu);
4342	AssertRCReturn(rc, rc);
4343	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4344
4345	/* Ensure we've cached (and hopefully modified) the VMCB for execution using hardware-assisted SVM. */
4346	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4347
4348	/*
4349	* No longjmps to ring-3 from this point on!!!
4350	*
4351	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4352	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4353	*/
4354	VMMRZCallRing3Disable(pVCpu);
4355
4356	/*
4357	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4358	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4359	* preemption disabled for a while. Since this is purly to aid the
4360	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4361	* disable interrupt on NT.
4362	*
4363	* We need to check for force-flags that could've possible been altered since we last
4364	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4365	* see @bugref{6398}).
4366	*
4367	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4368	* to ring-3 before executing guest code.
4369	*/
4370	pSvmTransient->fEFlags = ASMIntDisableFlags();
4371	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4372	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4373	{
4374	ASMSetFlags(pSvmTransient->fEFlags);
4375	VMMRZCallRing3Enable(pVCpu);
4376	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4377	return VINF_EM_RAW_TO_R3;
4378	}
4379	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4380	{
4381	ASMSetFlags(pSvmTransient->fEFlags);
4382	VMMRZCallRing3Enable(pVCpu);
4383	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4384	return VINF_EM_RAW_INTERRUPT;
4385	}
4386	return VINF_SUCCESS;
4387	}
4388	#endif
4389
4390
4391	/**
4392	* Does the preparations before executing guest code in AMD-V.
4393	*
4394	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4395	* recompiler. We must be cautious what we do here regarding committing
4396	* guest-state information into the VMCB assuming we assuredly execute the guest
4397	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4398	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4399	* that the recompiler can (and should) use them when it resumes guest
4400	* execution. Otherwise such operations must be done when we can no longer
4401	* exit to ring-3.
4402	*
4403	* @returns VBox status code (informational status codes included).
4404	* @retval VINF_SUCCESS if we can proceed with running the guest.
4405	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4406	*
4407	* @param pVCpu The cross context virtual CPU structure.
4408	* @param pSvmTransient Pointer to the SVM transient structure.
4409	*/
4410	static int hmR0SvmPreRunGuest(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
4411	{
4412	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4413	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4414
4415	/* Check force flag actions that might require us to go back to ring-3. */
4416	int rc = hmR0SvmCheckForceFlags(pVCpu);
4417	if (rc != VINF_SUCCESS)
4418	return rc;
4419
4420	if (TRPMHasTrap(pVCpu))
4421	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4422	else if (!pVCpu->hm.s.Event.fPending)
4423	hmR0SvmEvaluatePendingEvent(pVCpu);
4424
4425	/*
4426	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4427	* Just do it in software, see @bugref{8411}.
4428	* NB: If we could continue a task switch exit we wouldn't need to do this.
4429	*/
4430	PVM pVM = pVCpu->CTX_SUFF(pVM);
4431	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending && (((pVCpu->hm.s.Event.u64IntInfo >> 8) & 7) == SVM_EVENT_NMI)))
4432	if (RT_UNLIKELY(!pVM->hm.s.svm.u32Features))
4433	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4434
4435	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4436	Assert(!(pVCpu->cpum.GstCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4437	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4438	#endif
4439
4440	/*
4441	* Export the guest state bits that are not shared with the host in any way as we can
4442	* longjmp or get preempted in the midst of exporting some of the state.
4443	*/
4444	rc = hmR0SvmExportGuestState(pVCpu);
4445	AssertRCReturn(rc, rc);
4446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4447
4448	/*
4449	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4450	* world-switch so we can update it on the way back if the guest changed the TPR.
4451	*/
4452	if (pVCpu->hm.s.svm.fSyncVTpr)
4453	{
4454	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4455	if (pVM->hm.s.fTPRPatchingActive)
4456	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4457	else
4458	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4459	}
4460
4461	/*
4462	* No longjmps to ring-3 from this point on!!!
4463	*
4464	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4465	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4466	*/
4467	VMMRZCallRing3Disable(pVCpu);
4468
4469	/*
4470	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4471	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4472	* preemption disabled for a while. Since this is purly to aid the
4473	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4474	* disable interrupt on NT.
4475	*
4476	* We need to check for force-flags that could've possible been altered since we last
4477	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4478	* see @bugref{6398}).
4479	*
4480	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4481	* to ring-3 before executing guest code.
4482	*/
4483	pSvmTransient->fEFlags = ASMIntDisableFlags();
4484	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4485	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4486	{
4487	ASMSetFlags(pSvmTransient->fEFlags);
4488	VMMRZCallRing3Enable(pVCpu);
4489	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4490	return VINF_EM_RAW_TO_R3;
4491	}
4492	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4493	{
4494	ASMSetFlags(pSvmTransient->fEFlags);
4495	VMMRZCallRing3Enable(pVCpu);
4496	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4497	return VINF_EM_RAW_INTERRUPT;
4498	}
4499
4500	return VINF_SUCCESS;
4501	}
4502
4503
4504	/**
4505	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4506	* doing so.
4507	*
4508	* This means there is no backing out to ring-3 or anywhere else at this point.
4509	*
4510	* @param pVCpu The cross context virtual CPU structure.
4511	* @param pSvmTransient Pointer to the SVM transient structure.
4512	*
4513	* @remarks Called with preemption disabled.
4514	* @remarks No-long-jump zone!!!
4515	*/
4516	static void hmR0SvmPreRunGuestCommitted(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
4517	{
4518	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4519	Assert(VMMR0IsLogFlushDisabled(pVCpu));
4520	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4521
4522	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4523	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4524
4525	PVM pVM = pVCpu->CTX_SUFF(pVM);
4526	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4527
4528	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4529
4530	if (!CPUMIsGuestFPUStateActive(pVCpu))
4531	{
4532	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4533	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4534	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4535	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4536	}
4537
4538	/* Load the state shared between host and guest (FPU, debug). */
4539	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4540	hmR0SvmExportSharedState(pVCpu, pVmcb);
4541
4542	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4543	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4544
4545	PHMGLOBALCPUINFO pHostCpu = hmR0GetCurrentCpu();
4546	RTCPUID const idHostCpu = pHostCpu->idCpu;
4547	bool const fMigratedHostCpu = idHostCpu != pVCpu->hm.s.idLastCpu;
4548
4549	/* Setup TSC offsetting. */
4550	if ( pSvmTransient->fUpdateTscOffsetting
4551	\|\| fMigratedHostCpu)
4552	{
4553	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4554	pSvmTransient->fUpdateTscOffsetting = false;
4555	}
4556
4557	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4558	if (fMigratedHostCpu)
4559	pVmcb->ctrl.u32VmcbCleanBits = 0;
4560
4561	/* Store status of the shared guest-host state at the time of VMRUN. */
4562	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4563	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4564	{
4565	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4566	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4567	}
4568	else
4569	#endif
4570	{
4571	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4572	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4573	}
4574
4575	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4576	uint8_t *pbMsrBitmap;
4577	if (!pSvmTransient->fIsNestedGuest)
4578	pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
4579	else
4580	{
4581	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4582
4583	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4584	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4585	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4586	}
4587	#else
4588	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4589	#endif
4590
4591	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4592	/* Flush the appropriate tagged-TLB entries. */
4593	hmR0SvmFlushTaggedTlb(pVCpu, pVmcb, pHostCpu);
4594	Assert(pVCpu->hm.s.idLastCpu == idHostCpu);
4595
4596	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4597
4598	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4599	to start executing. */
4600
4601	/*
4602	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4603	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4604	*
4605	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4606	*/
4607	if ( pVM->cpum.ro.HostFeatures.fRdTscP
4608	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4609	{
4610	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4611	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4612	if (uGuestTscAux != pVCpu->hm.s.u64HostTscAux)
4613	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4614	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4615	pSvmTransient->fRestoreTscAuxMsr = true;
4616	}
4617	else
4618	{
4619	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4620	pSvmTransient->fRestoreTscAuxMsr = false;
4621	}
4622	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4623
4624	/*
4625	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4626	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4627	* from the VMCB.
4628	*/
4629	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu);
4630	if (!fSupportsVmcbCleanBits)
4631	pVmcb->ctrl.u32VmcbCleanBits = 0;
4632	}
4633
4634
4635	/**
4636	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4637	*
4638	* @returns VBox strict status code.
4639	* @param pVCpu The cross context virtual CPU structure.
4640	* @param HCPhysVmcb The host physical address of the VMCB.
4641	*
4642	* @remarks No-long-jump zone!!!
4643	*/
4644	DECLINLINE(int) hmR0SvmRunGuest(PVMCPU pVCpu, RTHCPHYS HCPhysVmcb)
4645	{
4646	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4647	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4648	pCtx->fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4649
4650	/*
4651	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4652	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4653	* callee-saved and thus the need for this XMM wrapper.
4654	*
4655	* Refer MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4656	*/
4657	PVM pVM = pVCpu->CTX_SUFF(pVM);
4658	#ifdef VBOX_WITH_KERNEL_USING_XMM
4659	return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, HCPhysVmcb, pCtx, pVM, pVCpu, pVCpu->hm.s.svm.pfnVMRun);
4660	#else
4661	return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, HCPhysVmcb, pCtx, pVM, pVCpu);
4662	#endif
4663	}
4664
4665
4666	/**
4667	* Undoes the TSC offset applied for an SVM nested-guest and returns the TSC
4668	* value for the guest.
4669	*
4670	* @returns The TSC offset after undoing any nested-guest TSC offset.
4671	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4672	* @param uTicks The nested-guest TSC.
4673	*
4674	* @note If you make any changes to this function, please check if
4675	* hmR0SvmNstGstUndoTscOffset() needs adjusting.
4676	*
4677	* @sa HMSvmNstGstApplyTscOffset().
4678	*/
4679	DECLINLINE(uint64_t) hmR0SvmNstGstUndoTscOffset(PVMCPU pVCpu, uint64_t uTicks)
4680	{
4681	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
4682	Assert(pVmcbNstGstCache->fCacheValid);
4683	return uTicks - pVmcbNstGstCache->u64TSCOffset;
4684	}
4685
4686
4687	/**
4688	* Performs some essential restoration of state after running guest (or
4689	* nested-guest) code in AMD-V.
4690	*
4691	* @param pVCpu The cross context virtual CPU structure.
4692	* @param pSvmTransient Pointer to the SVM transient structure.
4693	* @param rcVMRun Return code of VMRUN.
4694	*
4695	* @remarks Called with interrupts disabled.
4696	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4697	* unconditionally when it is safe to do so.
4698	*/
4699	static void hmR0SvmPostRunGuest(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4700	{
4701	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4702
4703	uint64_t const uHostTsc = ASMReadTSC(); /* Read the TSC as soon as possible. */
4704	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4705	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4706
4707	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4708	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4709
4710	/* TSC read must be done early for maximum accuracy. */
4711	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4712	{
4713	if (!pSvmTransient->fIsNestedGuest)
4714	TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
4715	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4716	else
4717	{
4718	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMSvmNstGstVmExitNotify(). */
4719	uint64_t const uGstTsc = hmR0SvmNstGstUndoTscOffset(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
4720	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4721	}
4722	#endif
4723	}
4724
4725	if (pSvmTransient->fRestoreTscAuxMsr)
4726	{
4727	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4728	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4729	if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4730	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4731	}
4732
4733	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4734	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4735	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4736
4737	Assert(!(ASMGetFlags() & X86_EFL_IF));
4738	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4739	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4740
4741	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4742	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4743	{
4744	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4745	return;
4746	}
4747
4748	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4749	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4750	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4751	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4752
4753	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4754	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4755	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4756	#else
4757	/*
4758	* Always import the following:
4759	*
4760	* - RIP for exit optimizations and evaluating event injection on re-entry.
4761	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4762	* state on preemption.
4763	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4764	* - CS for exit optimizations.
4765	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4766	* assembly switcher code.
4767	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4768	*/
4769	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4770	\| CPUMCTX_EXTRN_RFLAGS
4771	\| CPUMCTX_EXTRN_RAX
4772	\| CPUMCTX_EXTRN_RSP
4773	\| CPUMCTX_EXTRN_CS
4774	\| CPUMCTX_EXTRN_HWVIRT
4775	\| CPUMCTX_EXTRN_HM_SVM_INT_SHADOW
4776	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4777	\| HMSVM_CPUMCTX_SHARED_STATE);
4778	#endif
4779
4780	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4781	&& pVCpu->hm.s.svm.fSyncVTpr)
4782	{
4783	Assert(!pSvmTransient->fIsNestedGuest);
4784	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4785	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive
4786	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4787	{
4788	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4789	AssertRC(rc);
4790	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4791	}
4792	/* Sync TPR when we aren't intercepting CR8 writes. */
4793	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4794	{
4795	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4796	AssertRC(rc);
4797	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4798	}
4799	}
4800
4801	#ifdef DEBUG_ramshankar
4802	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4803	{
4804	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4805	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4806	0 /* uVerbose */);
4807	}
4808	#endif
4809
4810	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4811	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4812	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
4813	}
4814
4815
4816	/**
4817	* Runs the guest code using AMD-V.
4818	*
4819	* @returns VBox status code.
4820	* @param pVCpu The cross context virtual CPU structure.
4821	* @param pcLoops Pointer to the number of executed loops.
4822	*/
4823	static int hmR0SvmRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
4824	{
4825	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
4826	Assert(pcLoops);
4827	Assert(*pcLoops <= cMaxResumeLoops);
4828
4829	SVMTRANSIENT SvmTransient;
4830	RT_ZERO(SvmTransient);
4831	SvmTransient.fUpdateTscOffsetting = true;
4832	SvmTransient.pVmcb = pVCpu->hm.s.svm.pVmcb;
4833
4834	int rc = VERR_INTERNAL_ERROR_5;
4835	for (;;)
4836	{
4837	Assert(!HMR0SuspendPending());
4838	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4839
4840	/* Preparatory work for running nested-guest code, this may force us to return to
4841	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4842	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4843	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4844	if (rc != VINF_SUCCESS)
4845	break;
4846
4847	/*
4848	* No longjmps to ring-3 from this point on!!!
4849	*
4850	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4851	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4852	*/
4853	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4854	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hm.s.svm.HCPhysVmcb);
4855
4856	/* Restore any residual host-state and save any bits shared between host and guest
4857	into the guest-CPU state. Re-enables interrupts! */
4858	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4859
4860	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4861	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4862	{
4863	if (rc == VINF_SUCCESS)
4864	rc = VERR_SVM_INVALID_GUEST_STATE;
4865	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4866	hmR0SvmReportWorldSwitchError(pVCpu, rc);
4867	break;
4868	}
4869
4870	/* Handle the #VMEXIT. */
4871	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4872	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4873	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4874	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4875	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4876	if (rc != VINF_SUCCESS)
4877	break;
4878	if (++(*pcLoops) >= cMaxResumeLoops)
4879	{
4880	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4881	rc = VINF_EM_RAW_INTERRUPT;
4882	break;
4883	}
4884	}
4885
4886	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4887	return rc;
4888	}
4889
4890
4891	/**
4892	* Runs the guest code using AMD-V in single step mode.
4893	*
4894	* @returns VBox status code.
4895	* @param pVCpu The cross context virtual CPU structure.
4896	* @param pcLoops Pointer to the number of executed loops.
4897	*/
4898	static int hmR0SvmRunGuestCodeStep(PVMCPU pVCpu, uint32_t *pcLoops)
4899	{
4900	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
4901	Assert(pcLoops);
4902	Assert(*pcLoops <= cMaxResumeLoops);
4903
4904	SVMTRANSIENT SvmTransient;
4905	RT_ZERO(SvmTransient);
4906	SvmTransient.fUpdateTscOffsetting = true;
4907	SvmTransient.pVmcb = pVCpu->hm.s.svm.pVmcb;
4908
4909	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4910	uint16_t uCsStart = pCtx->cs.Sel;
4911	uint64_t uRipStart = pCtx->rip;
4912
4913	int rc = VERR_INTERNAL_ERROR_5;
4914	for (;;)
4915	{
4916	Assert(!HMR0SuspendPending());
4917	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
4918	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
4919	(unsigned)RTMpCpuId(), *pcLoops));
4920
4921	/* Preparatory work for running nested-guest code, this may force us to return to
4922	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4923	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4924	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4925	if (rc != VINF_SUCCESS)
4926	break;
4927
4928	/*
4929	* No longjmps to ring-3 from this point on!!!
4930	*
4931	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4932	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4933	*/
4934	VMMRZCallRing3Disable(pVCpu);
4935	VMMRZCallRing3RemoveNotification(pVCpu);
4936	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4937
4938	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hm.s.svm.HCPhysVmcb);
4939
4940	/* Restore any residual host-state and save any bits shared between host and guest
4941	into the guest-CPU state. Re-enables interrupts! */
4942	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4943
4944	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4945	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4946	{
4947	if (rc == VINF_SUCCESS)
4948	rc = VERR_SVM_INVALID_GUEST_STATE;
4949	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4950	hmR0SvmReportWorldSwitchError(pVCpu, rc);
4951	return rc;
4952	}
4953
4954	/* Handle the #VMEXIT. */
4955	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4956	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4957	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4958	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4959	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4960	if (rc != VINF_SUCCESS)
4961	break;
4962	if (++(*pcLoops) >= cMaxResumeLoops)
4963	{
4964	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4965	rc = VINF_EM_RAW_INTERRUPT;
4966	break;
4967	}
4968
4969	/*
4970	* Did the RIP change, if so, consider it a single step.
4971	* Otherwise, make sure one of the TFs gets set.
4972	*/
4973	if ( pCtx->rip != uRipStart
4974	\|\| pCtx->cs.Sel != uCsStart)
4975	{
4976	rc = VINF_EM_DBG_STEPPED;
4977	break;
4978	}
4979	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR_MASK;
4980	}
4981
4982	/*
4983	* Clear the X86_EFL_TF if necessary.
4984	*/
4985	if (pVCpu->hm.s.fClearTrapFlag)
4986	{
4987	pVCpu->hm.s.fClearTrapFlag = false;
4988	pCtx->eflags.Bits.u1TF = 0;
4989	}
4990
4991	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4992	return rc;
4993	}
4994
4995	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4996	/**
4997	* Runs the nested-guest code using AMD-V.
4998	*
4999	* @returns VBox status code.
5000	* @param pVCpu The cross context virtual CPU structure.
5001	* @param pcLoops Pointer to the number of executed loops. If we're switching
5002	* from the guest-code execution loop to this nested-guest
5003	* execution loop pass the remainder value, else pass 0.
5004	*/
5005	static int hmR0SvmRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
5006	{
5007	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5008	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
5009	Assert(pcLoops);
5010	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops);
5011
5012	SVMTRANSIENT SvmTransient;
5013	RT_ZERO(SvmTransient);
5014	SvmTransient.fUpdateTscOffsetting = true;
5015	SvmTransient.pVmcb = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
5016	SvmTransient.fIsNestedGuest = true;
5017
5018	int rc = VERR_INTERNAL_ERROR_4;
5019	for (;;)
5020	{
5021	Assert(!HMR0SuspendPending());
5022	HMSVM_ASSERT_CPU_SAFE(pVCpu);
5023
5024	/* Preparatory work for running nested-guest code, this may force us to return to
5025	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
5026	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
5027	rc = hmR0SvmPreRunGuestNested(pVCpu, &SvmTransient);
5028	if ( rc != VINF_SUCCESS
5029	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
5030	{
5031	break;
5032	}
5033
5034	/*
5035	* No longjmps to ring-3 from this point on!!!
5036	*
5037	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
5038	* better than a kernel panic. This also disables flushing of the R0-logger instance.
5039	*/
5040	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
5041
5042	rc = hmR0SvmRunGuest(pVCpu, pCtx->hwvirt.svm.HCPhysVmcb);
5043
5044	/* Restore any residual host-state and save any bits shared between host and guest
5045	into the guest-CPU state. Re-enables interrupts! */
5046	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
5047
5048	if (RT_LIKELY( rc == VINF_SUCCESS
5049	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
5050	{ /* extremely likely */ }
5051	else
5052	{
5053	/* VMRUN failed, shouldn't really happen, Guru. */
5054	if (rc != VINF_SUCCESS)
5055	break;
5056
5057	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
5058	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
5059	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
5060	rc = VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0));
5061	break;
5062	}
5063
5064	/* Handle the #VMEXIT. */
5065	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
5066	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
5067	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
5068	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
5069	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
5070	if ( rc != VINF_SUCCESS
5071	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
5072	break;
5073	if (++(*pcLoops) >= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
5074	{
5075	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
5076	rc = VINF_EM_RAW_INTERRUPT;
5077	break;
5078	}
5079
5080	/** @todo handle single-stepping */
5081	}
5082
5083	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5084	return rc;
5085	}
5086	#endif
5087
5088
5089	/**
5090	* Runs the guest code using AMD-V.
5091	*
5092	* @returns Strict VBox status code.
5093	* @param pVCpu The cross context virtual CPU structure.
5094	*/
5095	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPU pVCpu)
5096	{
5097	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5098	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
5099	VMMRZCallRing3SetNotification(pVCpu, hmR0SvmCallRing3Callback, NULL /* pvUser */);
5100
5101	uint32_t cLoops = 0;
5102	int rc;
5103	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5104	if (!CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
5105	#endif
5106	{
5107	if (!pVCpu->hm.s.fSingleInstruction)
5108	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
5109	else
5110	rc = hmR0SvmRunGuestCodeStep(pVCpu, &cLoops);
5111	}
5112	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5113	else
5114	{
5115	rc = VINF_SVM_VMRUN;
5116	}
5117
5118	/* Re-check the nested-guest condition here as we may be transitioning from the normal
5119	execution loop into the nested-guest, hence this is not placed in the 'else' part above. */
5120	if (rc == VINF_SVM_VMRUN)
5121	{
5122	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
5123	if (rc == VINF_SVM_VMEXIT)
5124	rc = VINF_SUCCESS;
5125	}
5126	#endif
5127
5128	/* Fixup error codes. */
5129	if (rc == VERR_EM_INTERPRETER)
5130	rc = VINF_EM_RAW_EMULATE_INSTR;
5131	else if (rc == VINF_EM_RESET)
5132	rc = VINF_EM_TRIPLE_FAULT;
5133
5134	/* Prepare to return to ring-3. This will remove longjmp notifications. */
5135	rc = hmR0SvmExitToRing3(pVCpu, rc);
5136	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
5137	return rc;
5138	}
5139
5140
5141	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5142	/**
5143	* Determines whether an IOIO intercept is active for the nested-guest or not.
5144	*
5145	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
5146	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
5147	*/
5148	static bool hmR0SvmIsIoInterceptActive(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
5149	{
5150	const uint16_t u16Port = pIoExitInfo->n.u16Port;
5151	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
5152	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
5153	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
5154	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
5155	const bool fRep = pIoExitInfo->n.u1Rep;
5156	const bool fStrIo = pIoExitInfo->n.u1Str;
5157
5158	return HMSvmIsIOInterceptActive(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
5159	NULL /* pIoExitInfo */);
5160	}
5161
5162
5163	/**
5164	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
5165	* SVM_EXIT_INVALID).
5166	*
5167	* @returns VBox status code (informational status codes included).
5168	* @param pVCpu The cross context virtual CPU structure.
5169	* @param pSvmTransient Pointer to the SVM transient structure.
5170	*/
5171	static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
5172	{
5173	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
5174	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5175	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5176
5177	/*
5178	* We import the complete state here because we use separate VMCBs for the guest and the
5179	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
5180	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
5181	*/
5182	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
5183	do { \
5184	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5185	return VBOXSTRICTRC_TODO(IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2))); \
5186	} while (0)
5187
5188	/*
5189	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
5190	* nested-guest. If it isn't, it should be handled by the (outer) guest.
5191	*/
5192	PSVMVMCB pVmcbNstGst = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
5193	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
5194	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
5195	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
5196	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
5197
5198	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
5199	switch (uExitCode)
5200	{
5201	case SVM_EXIT_CPUID:
5202	{
5203	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_CPUID))
5204	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5205	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
5206	}
5207
5208	case SVM_EXIT_RDTSC:
5209	{
5210	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_RDTSC))
5211	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5212	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
5213	}
5214
5215	case SVM_EXIT_RDTSCP:
5216	{
5217	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_RDTSCP))
5218	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5219	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
5220	}
5221
5222	case SVM_EXIT_MONITOR:
5223	{
5224	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_MONITOR))
5225	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5226	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
5227	}
5228
5229	case SVM_EXIT_MWAIT:
5230	{
5231	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_MWAIT))
5232	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5233	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
5234	}
5235
5236	case SVM_EXIT_HLT:
5237	{
5238	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_HLT))
5239	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5240	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
5241	}
5242
5243	case SVM_EXIT_MSR:
5244	{
5245	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
5246	{
5247	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
5248	uint16_t offMsrpm;
5249	uint8_t uMsrpmBit;
5250	int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
5251	if (RT_SUCCESS(rc))
5252	{
5253	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
5254	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
5255
5256	uint8_t const pbMsrBitmap = (uint8_t const )pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
5257	pbMsrBitmap += offMsrpm;
5258	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
5259	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
5260
5261	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
5262	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
5263	{
5264	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5265	}
5266	}
5267	else
5268	{
5269	/*
5270	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
5271	* See AMD-V spec. "15.11 MSR Intercepts".
5272	*/
5273	Assert(rc == VERR_OUT_OF_RANGE);
5274	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5275	}
5276	}
5277	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
5278	}
5279
5280	case SVM_EXIT_IOIO:
5281	{
5282	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
5283	{
5284	void *pvIoBitmap = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvIoBitmap);
5285	SVMIOIOEXITINFO IoExitInfo;
5286	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
5287	bool const fIntercept = hmR0SvmIsIoInterceptActive(pvIoBitmap, &IoExitInfo);
5288	if (fIntercept)
5289	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5290	}
5291	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
5292	}
5293
5294	case SVM_EXIT_XCPT_PF:
5295	{
5296	PVM pVM = pVCpu->CTX_SUFF(pVM);
5297	if (pVM->hm.s.fNestedPaging)
5298	{
5299	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
5300	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
5301
5302	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
5303	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_PF))
5304	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
5305
5306	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
5307	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
5308	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
5309	return VINF_SUCCESS;
5310	}
5311	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
5312	}
5313
5314	case SVM_EXIT_XCPT_UD:
5315	{
5316	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_UD))
5317	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5318	hmR0SvmSetPendingXcptUD(pVCpu);
5319	return VINF_SUCCESS;
5320	}
5321
5322	case SVM_EXIT_XCPT_MF:
5323	{
5324	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_MF))
5325	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5326	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
5327	}
5328
5329	case SVM_EXIT_XCPT_DB:
5330	{
5331	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_DB))
5332	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5333	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
5334	}
5335
5336	case SVM_EXIT_XCPT_AC:
5337	{
5338	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_AC))
5339	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5340	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5341	}
5342
5343	case SVM_EXIT_XCPT_BP:
5344	{
5345	if (HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_BP))
5346	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5347	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5348	}
5349
5350	case SVM_EXIT_READ_CR0:
5351	case SVM_EXIT_READ_CR3:
5352	case SVM_EXIT_READ_CR4:
5353	{
5354	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5355	if (HMIsGuestSvmReadCRxInterceptSet(pVCpu, uCr))
5356	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5357	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5358	}
5359
5360	case SVM_EXIT_CR0_SEL_WRITE:
5361	{
5362	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5363	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5364	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5365	}
5366
5367	case SVM_EXIT_WRITE_CR0:
5368	case SVM_EXIT_WRITE_CR3:
5369	case SVM_EXIT_WRITE_CR4:
5370	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5371	{
5372	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5373	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5374
5375	if (HMIsGuestSvmWriteCRxInterceptSet(pVCpu, uCr))
5376	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5377	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5378	}
5379
5380	case SVM_EXIT_PAUSE:
5381	{
5382	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_PAUSE))
5383	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5384	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5385	}
5386
5387	case SVM_EXIT_VINTR:
5388	{
5389	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VINTR))
5390	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5391	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5392	}
5393
5394	case SVM_EXIT_INTR:
5395	case SVM_EXIT_NMI:
5396	case SVM_EXIT_SMI:
5397	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5398	{
5399	/*
5400	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5401	*
5402	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5403	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5404	* nested-guest #VMEXIT.
5405	*
5406	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5407	* while trying to inject interrupts, see comment at the top of this function.
5408	*/
5409	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5410	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5411	}
5412
5413	case SVM_EXIT_FERR_FREEZE:
5414	{
5415	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5416	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5417	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5418	}
5419
5420	case SVM_EXIT_INVLPG:
5421	{
5422	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_INVLPG))
5423	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5424	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5425	}
5426
5427	case SVM_EXIT_WBINVD:
5428	{
5429	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_WBINVD))
5430	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5431	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5432	}
5433
5434	case SVM_EXIT_INVD:
5435	{
5436	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_INVD))
5437	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5438	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5439	}
5440
5441	case SVM_EXIT_RDPMC:
5442	{
5443	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_RDPMC))
5444	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5445	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5446	}
5447
5448	default:
5449	{
5450	switch (uExitCode)
5451	{
5452	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5453	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5454	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5455	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5456	{
5457	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5458	if (HMIsGuestSvmReadDRxInterceptSet(pVCpu, uDr))
5459	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5460	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5461	}
5462
5463	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5464	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5465	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5466	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5467	{
5468	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5469	if (HMIsGuestSvmWriteDRxInterceptSet(pVCpu, uDr))
5470	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5471	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5472	}
5473
5474	case SVM_EXIT_XCPT_DE:
5475	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5476	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5477	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5478	case SVM_EXIT_XCPT_OF:
5479	case SVM_EXIT_XCPT_BR:
5480	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5481	case SVM_EXIT_XCPT_NM:
5482	case SVM_EXIT_XCPT_DF:
5483	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5484	case SVM_EXIT_XCPT_TS:
5485	case SVM_EXIT_XCPT_NP:
5486	case SVM_EXIT_XCPT_SS:
5487	case SVM_EXIT_XCPT_GP:
5488	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5489	case SVM_EXIT_XCPT_15: /* Reserved. */
5490	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5491	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5492	case SVM_EXIT_XCPT_MC:
5493	case SVM_EXIT_XCPT_XF:
5494	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5495	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5496	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5497	{
5498	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5499	if (HMIsGuestSvmXcptInterceptSet(pVCpu, uVector))
5500	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5501	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5502	}
5503
5504	case SVM_EXIT_XSETBV:
5505	{
5506	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_XSETBV))
5507	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5508	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5509	}
5510
5511	case SVM_EXIT_TASK_SWITCH:
5512	{
5513	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5514	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5515	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5516	}
5517
5518	case SVM_EXIT_IRET:
5519	{
5520	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_IRET))
5521	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5522	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5523	}
5524
5525	case SVM_EXIT_SHUTDOWN:
5526	{
5527	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
5528	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5529	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5530	}
5531
5532	case SVM_EXIT_VMMCALL:
5533	{
5534	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VMMCALL))
5535	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5536	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5537	}
5538
5539	case SVM_EXIT_CLGI:
5540	{
5541	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_CLGI))
5542	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5543	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5544	}
5545
5546	case SVM_EXIT_STGI:
5547	{
5548	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_STGI))
5549	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5550	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5551	}
5552
5553	case SVM_EXIT_VMLOAD:
5554	{
5555	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VMLOAD))
5556	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5557	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5558	}
5559
5560	case SVM_EXIT_VMSAVE:
5561	{
5562	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VMSAVE))
5563	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5564	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5565	}
5566
5567	case SVM_EXIT_INVLPGA:
5568	{
5569	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_INVLPGA))
5570	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5571	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5572	}
5573
5574	case SVM_EXIT_VMRUN:
5575	{
5576	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_VMRUN))
5577	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5578	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5579	}
5580
5581	case SVM_EXIT_RSM:
5582	{
5583	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_RSM))
5584	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5585	hmR0SvmSetPendingXcptUD(pVCpu);
5586	return VINF_SUCCESS;
5587	}
5588
5589	case SVM_EXIT_SKINIT:
5590	{
5591	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, SVM_CTRL_INTERCEPT_SKINIT))
5592	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5593	hmR0SvmSetPendingXcptUD(pVCpu);
5594	return VINF_SUCCESS;
5595	}
5596
5597	case SVM_EXIT_NPF:
5598	{
5599	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
5600	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5601	}
5602
5603	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5604	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5605
5606	default:
5607	{
5608	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5609	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5610	return VERR_SVM_UNKNOWN_EXIT;
5611	}
5612	}
5613	}
5614	}
5615	/* not reached */
5616
5617	#undef NST_GST_VMEXIT_CALL_RET
5618	}
5619	#endif
5620
5621
5622	/**
5623	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5624	*
5625	* @returns VBox status code (informational status codes included).
5626	* @param pVCpu The cross context virtual CPU structure.
5627	* @param pSvmTransient Pointer to the SVM transient structure.
5628	*/
5629	static int hmR0SvmHandleExit(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
5630	{
5631	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5632	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5633
5634	#ifdef DEBUG_ramshankar
5635	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5636	do { \
5637	if ((a_fDbg) == 1) \
5638	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5639	int rc = a_CallExpr; \
5640	if ((a_fDbg) == 1) \
5641	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5642	return rc; \
5643	} while (0)
5644	#else
5645	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5646	#endif
5647
5648	/*
5649	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5650	* for most guests under normal workloads (for some definition of "normal").
5651	*/
5652	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5653	switch (uExitCode)
5654	{
5655	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5656	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5657	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5658	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5659	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5660	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5661	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5662	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5663	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5664	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5665
5666	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5667	case SVM_EXIT_INTR:
5668	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5669
5670	case SVM_EXIT_READ_CR0:
5671	case SVM_EXIT_READ_CR3:
5672	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5673
5674	case SVM_EXIT_CR0_SEL_WRITE:
5675	case SVM_EXIT_WRITE_CR0:
5676	case SVM_EXIT_WRITE_CR3:
5677	case SVM_EXIT_WRITE_CR4:
5678	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5679
5680	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5681	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5682	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5683	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5684	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5685	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5686	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5687	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5688	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5689	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5690	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5691	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5692	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5693	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5694	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5695
5696	default:
5697	{
5698	switch (pSvmTransient->u64ExitCode)
5699	{
5700	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5701	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5702	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5703	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5704	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5705
5706	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5707	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5708	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5709	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5710	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5711
5712	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5713	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5714
5715	case SVM_EXIT_SMI:
5716	case SVM_EXIT_INIT:
5717	{
5718	/*
5719	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5720	* If it ever does, we want to know about it so log the exit code and bail.
5721	*/
5722	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5723	}
5724
5725	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5726	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5727	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5728	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5729	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5730	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5731	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5732	#else
5733	case SVM_EXIT_CLGI:
5734	case SVM_EXIT_STGI:
5735	case SVM_EXIT_VMLOAD:
5736	case SVM_EXIT_VMSAVE:
5737	case SVM_EXIT_INVLPGA:
5738	case SVM_EXIT_VMRUN:
5739	#endif
5740	case SVM_EXIT_RSM:
5741	case SVM_EXIT_SKINIT:
5742	{
5743	hmR0SvmSetPendingXcptUD(pVCpu);
5744	return VINF_SUCCESS;
5745	}
5746
5747	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
5748	case SVM_EXIT_XCPT_DE:
5749	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5750	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5751	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5752	case SVM_EXIT_XCPT_OF:
5753	case SVM_EXIT_XCPT_BR:
5754	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5755	case SVM_EXIT_XCPT_NM:
5756	case SVM_EXIT_XCPT_DF:
5757	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5758	case SVM_EXIT_XCPT_TS:
5759	case SVM_EXIT_XCPT_NP:
5760	case SVM_EXIT_XCPT_SS:
5761	case SVM_EXIT_XCPT_GP:
5762	/* SVM_EXIT_XCPT_PF: */
5763	case SVM_EXIT_XCPT_15: /* Reserved. */
5764	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5765	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5766	case SVM_EXIT_XCPT_MC:
5767	case SVM_EXIT_XCPT_XF:
5768	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5769	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5770	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5771	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5772	#endif /* HMSVM_ALWAYS_TRAP_ALL_XCPTS */
5773
5774	default:
5775	{
5776	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5777	pVCpu->hm.s.u32HMError = uExitCode;
5778	return VERR_SVM_UNKNOWN_EXIT;
5779	}
5780	}
5781	}
5782	}
5783	/* not reached */
5784	#undef VMEXIT_CALL_RET
5785	}
5786
5787
5788	#ifdef VBOX_STRICT
5789	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5790	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
5791	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5792
5793	# define HMSVM_ASSERT_PREEMPT_CPUID() \
5794	do \
5795	{ \
5796	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5797	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5798	} while (0)
5799
5800	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
5801	do { \
5802	AssertPtr((a_pVCpu)); \
5803	AssertPtr((a_pSvmTransient)); \
5804	Assert(ASMIntAreEnabled()); \
5805	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
5806	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
5807	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
5808	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
5809	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
5810	HMSVM_ASSERT_PREEMPT_CPUID(); \
5811	} while (0)
5812	#else
5813	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
5814	do { \
5815	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
5816	} while (0)
5817	#endif
5818
5819
5820	/**
5821	* Gets the IEM exception flags for the specified SVM event.
5822	*
5823	* @returns The IEM exception flags.
5824	* @param pEvent Pointer to the SVM event.
5825	*
5826	* @remarks This function currently only constructs flags required for
5827	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
5828	* and CR2 aspects of an exception are not included).
5829	*/
5830	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
5831	{
5832	uint8_t const uEventType = pEvent->n.u3Type;
5833	uint32_t fIemXcptFlags;
5834	switch (uEventType)
5835	{
5836	case SVM_EVENT_EXCEPTION:
5837	/*
5838	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
5839	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
5840	*/
5841	if (pEvent->n.u8Vector == X86_XCPT_BP)
5842	{
5843	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
5844	break;
5845	}
5846	if (pEvent->n.u8Vector == X86_XCPT_OF)
5847	{
5848	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
5849	break;
5850	}
5851	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
5852	RT_FALL_THRU();
5853	case SVM_EVENT_NMI:
5854	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5855	break;
5856
5857	case SVM_EVENT_EXTERNAL_IRQ:
5858	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5859	break;
5860
5861	case SVM_EVENT_SOFTWARE_INT:
5862	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5863	break;
5864
5865	default:
5866	fIemXcptFlags = 0;
5867	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
5868	break;
5869	}
5870	return fIemXcptFlags;
5871	}
5872
5873
5874	/**
5875	* Handle a condition that occurred while delivering an event through the guest
5876	* IDT.
5877	*
5878	* @returns VBox status code (informational error codes included).
5879	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
5880	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
5881	* continue execution of the guest which will delivery the \#DF.
5882	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5883	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5884	*
5885	* @param pVCpu The cross context virtual CPU structure.
5886	* @param pSvmTransient Pointer to the SVM transient structure.
5887	*
5888	* @remarks No-long-jump zone!!!
5889	*/
5890	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
5891	{
5892	int rc = VINF_SUCCESS;
5893	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5894	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
5895
5896	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
5897	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
5898	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
5899
5900	/*
5901	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
5902	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
5903	*
5904	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
5905	*/
5906	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
5907	{
5908	IEMXCPTRAISE enmRaise;
5909	IEMXCPTRAISEINFO fRaiseInfo;
5910	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
5911	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5912	if (fExitIsHwXcpt)
5913	{
5914	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
5915	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
5916	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5917	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5918	}
5919	else
5920	{
5921	/*
5922	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
5923	* then we end up here.
5924	*
5925	* If the event was:
5926	* - a software interrupt, we can re-execute the instruction which will
5927	* regenerate the event.
5928	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
5929	* - a hardware exception or external interrupt, we re-inject it.
5930	*/
5931	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5932	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
5933	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5934	else
5935	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5936	}
5937
5938	switch (enmRaise)
5939	{
5940	case IEMXCPTRAISE_CURRENT_XCPT:
5941	case IEMXCPTRAISE_PREV_EVENT:
5942	{
5943	/* For software interrupts, we shall re-execute the instruction. */
5944	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
5945	{
5946	RTGCUINTPTR GCPtrFaultAddress = 0;
5947
5948	/* If we are re-injecting an NMI, clear NMI blocking. */
5949	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5950	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5951
5952	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
5953	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5954	{
5955	pSvmTransient->fVectoringPF = true;
5956	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
5957	pVCpu->cpum.GstCtx.cr2));
5958	}
5959	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
5960	&& uIdtVector == X86_XCPT_PF)
5961	{
5962	/*
5963	* If the previous exception was a #PF, we need to recover the CR2 value.
5964	* This can't happen with shadow paging.
5965	*/
5966	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
5967	}
5968
5969	/*
5970	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
5971	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
5972	*/
5973	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5974	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5975	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
5976
5977	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
5978	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
5979	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
5980	}
5981	break;
5982	}
5983
5984	case IEMXCPTRAISE_REEXEC_INSTR:
5985	{
5986	Assert(rc == VINF_SUCCESS);
5987	break;
5988	}
5989
5990	case IEMXCPTRAISE_DOUBLE_FAULT:
5991	{
5992	/*
5993	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
5994	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
5995	* converted into a #DF.
5996	*/
5997	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5998	{
5999	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6000	pSvmTransient->fVectoringDoublePF = true;
6001	Assert(rc == VINF_SUCCESS);
6002	}
6003	else
6004	{
6005	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6006	hmR0SvmSetPendingXcptDF(pVCpu);
6007	rc = VINF_HM_DOUBLE_FAULT;
6008	}
6009	break;
6010	}
6011
6012	case IEMXCPTRAISE_TRIPLE_FAULT:
6013	{
6014	rc = VINF_EM_RESET;
6015	break;
6016	}
6017
6018	case IEMXCPTRAISE_CPU_HANG:
6019	{
6020	rc = VERR_EM_GUEST_CPU_HANG;
6021	break;
6022	}
6023
6024	default:
6025	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6026	}
6027	}
6028	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6029	return rc;
6030	}
6031
6032
6033	/**
6034	* Advances the guest RIP by the number of bytes specified in @a cb.
6035	*
6036	* @param pVCpu The cross context virtual CPU structure.
6037	* @param cb RIP increment value in bytes.
6038	*/
6039	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPU pVCpu, uint32_t cb)
6040	{
6041	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6042	pCtx->rip += cb;
6043
6044	/* Update interrupt shadow. */
6045	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
6046	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
6047	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6048	}
6049
6050
6051	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6052	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6053	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6054
6055	/** @name \#VMEXIT handlers.
6056	* @{
6057	*/
6058
6059	/**
6060	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6061	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6062	*/
6063	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6064	{
6065	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6066
6067	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6068	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6069	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6070	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6071
6072	/*
6073	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6074	* signal -before- the timer fires if the current interrupt is our own timer or a some
6075	* other host interrupt. We also cannot examine what interrupt it is until the host
6076	* actually take the interrupt.
6077	*
6078	* Going back to executing guest code here unconditionally causes random scheduling
6079	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6080	*/
6081	return VINF_EM_RAW_INTERRUPT;
6082	}
6083
6084
6085	/**
6086	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6087	*/
6088	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6089	{
6090	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6091
6092	VBOXSTRICTRC rcStrict;
6093	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6094	if (fSupportsNextRipSave)
6095	{
6096	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6097	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6098	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6099	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6100	}
6101	else
6102	{
6103	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6104	rcStrict = IEMExecOne(pVCpu);
6105	}
6106
6107	if (rcStrict == VINF_IEM_RAISED_XCPT)
6108	{
6109	rcStrict = VINF_SUCCESS;
6110	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6111	}
6112	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6113	return VBOXSTRICTRC_TODO(rcStrict);
6114	}
6115
6116
6117	/**
6118	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6119	*/
6120	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6121	{
6122	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6123
6124	VBOXSTRICTRC rcStrict;
6125	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6126	if (fSupportsNextRipSave)
6127	{
6128	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6129	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6130	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6131	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6132	}
6133	else
6134	{
6135	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6136	rcStrict = IEMExecOne(pVCpu);
6137	}
6138
6139	if (rcStrict == VINF_IEM_RAISED_XCPT)
6140	{
6141	rcStrict = VINF_SUCCESS;
6142	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6143	}
6144	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6145	return VBOXSTRICTRC_TODO(rcStrict);
6146	}
6147
6148
6149	/**
6150	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6151	*/
6152	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6153	{
6154	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6155
6156	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6157	VBOXSTRICTRC rcStrict;
6158	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6159	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6160	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6161	if (!pExitRec)
6162	{
6163	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6164	if (fSupportsNextRipSave)
6165	{
6166	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6167	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6168	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6169	}
6170	else
6171	{
6172	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6173	rcStrict = IEMExecOne(pVCpu);
6174	}
6175
6176	if (rcStrict == VINF_IEM_RAISED_XCPT)
6177	{
6178	rcStrict = VINF_SUCCESS;
6179	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6180	}
6181	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6182	}
6183	else
6184	{
6185	/*
6186	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6187	*/
6188	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6189
6190	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6191	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6192
6193	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6194
6195	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6196	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6197	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6198	}
6199	return VBOXSTRICTRC_TODO(rcStrict);
6200	}
6201
6202
6203	/**
6204	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6205	*/
6206	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6207	{
6208	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6209
6210	VBOXSTRICTRC rcStrict;
6211	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6212	if (fSupportsNextRipSave)
6213	{
6214	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6215	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6216	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6217	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6218	}
6219	else
6220	{
6221	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6222	rcStrict = IEMExecOne(pVCpu);
6223	}
6224
6225	if (rcStrict == VINF_SUCCESS)
6226	pSvmTransient->fUpdateTscOffsetting = true;
6227	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6228	{
6229	rcStrict = VINF_SUCCESS;
6230	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6231	}
6232	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6233	return VBOXSTRICTRC_TODO(rcStrict);
6234	}
6235
6236
6237	/**
6238	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6239	*/
6240	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6241	{
6242	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6243
6244	VBOXSTRICTRC rcStrict;
6245	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6246	if (fSupportsNextRipSave)
6247	{
6248	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
6249	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6250	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6251	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
6252	}
6253	else
6254	{
6255	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6256	rcStrict = IEMExecOne(pVCpu);
6257	}
6258
6259	if (rcStrict == VINF_SUCCESS)
6260	pSvmTransient->fUpdateTscOffsetting = true;
6261	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6262	{
6263	rcStrict = VINF_SUCCESS;
6264	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6265	}
6266	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6267	return VBOXSTRICTRC_TODO(rcStrict);
6268	}
6269
6270
6271	/**
6272	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6273	*/
6274	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6275	{
6276	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6277
6278	VBOXSTRICTRC rcStrict;
6279	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6280	if (fSupportsNextRipSave)
6281	{
6282	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6283	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6284	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6285	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
6286	}
6287	else
6288	{
6289	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6290	rcStrict = IEMExecOne(pVCpu);
6291	}
6292
6293	if (rcStrict == VINF_IEM_RAISED_XCPT)
6294	{
6295	rcStrict = VINF_SUCCESS;
6296	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6297	}
6298	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6299	return VBOXSTRICTRC_TODO(rcStrict);
6300	}
6301
6302
6303	/**
6304	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
6305	*/
6306	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6307	{
6308	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6309	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
6310
6311	VBOXSTRICTRC rcStrict;
6312	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
6313	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6314	if ( fSupportsDecodeAssists
6315	&& fSupportsNextRipSave)
6316	{
6317	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
6318	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6319	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6320	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
6321	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
6322	}
6323	else
6324	{
6325	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6326	rcStrict = IEMExecOne(pVCpu);
6327	}
6328
6329	if (rcStrict == VINF_IEM_RAISED_XCPT)
6330	{
6331	rcStrict = VINF_SUCCESS;
6332	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6333	}
6334	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6335	return VBOXSTRICTRC_VAL(rcStrict);
6336	}
6337
6338
6339	/**
6340	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
6341	*/
6342	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6343	{
6344	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6345
6346	VBOXSTRICTRC rcStrict;
6347	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6348	if (fSupportsNextRipSave)
6349	{
6350	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6351	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6352	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6353	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
6354	}
6355	else
6356	{
6357	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6358	rcStrict = IEMExecOne(pVCpu);
6359	}
6360
6361	if ( rcStrict == VINF_EM_HALT
6362	\|\| rcStrict == VINF_SUCCESS)
6363	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
6364	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6365	{
6366	rcStrict = VINF_SUCCESS;
6367	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6368	}
6369	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6370	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
6371	if (rcStrict != VINF_SUCCESS)
6372	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
6373	return VBOXSTRICTRC_VAL(rcStrict);;
6374	}
6375
6376
6377	/**
6378	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
6379	*/
6380	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6381	{
6382	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6383
6384	/*
6385	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
6386	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
6387	* segment override prefix or other prefixes might be used, in which case we fallback to
6388	* IEMExecOne() to figure out.
6389	*/
6390	VBOXSTRICTRC rcStrict;
6391	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6392	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
6393	if (cbInstr)
6394	{
6395	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
6396	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
6397	}
6398	else
6399	{
6400	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6401	rcStrict = IEMExecOne(pVCpu);
6402	}
6403
6404	if (rcStrict == VINF_IEM_RAISED_XCPT)
6405	{
6406	rcStrict = VINF_SUCCESS;
6407	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6408	}
6409	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6410	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6411	return VBOXSTRICTRC_TODO(rcStrict);
6412	}
6413
6414
6415	/**
6416	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
6417	*/
6418	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6419	{
6420	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6421
6422	VBOXSTRICTRC rcStrict;
6423	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6424	if (fSupportsNextRipSave)
6425	{
6426	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6427	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6428	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6429	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
6430	}
6431	else
6432	{
6433	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6434	rcStrict = IEMExecOne(pVCpu);
6435	}
6436
6437	if ( rcStrict == VINF_EM_HALT
6438	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
6439	rcStrict = VINF_SUCCESS;
6440	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6441	{
6442	rcStrict = VINF_SUCCESS;
6443	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6444	}
6445	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
6447	return VBOXSTRICTRC_TODO(rcStrict);
6448	}
6449
6450
6451	/**
6452	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
6453	* \#VMEXIT.
6454	*/
6455	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6456	{
6457	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6458	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6459	return VINF_EM_RESET;
6460	}
6461
6462
6463	/**
6464	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
6465	*/
6466	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6467	{
6468	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6469	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6470	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
6471	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
6472	RT_NOREF(pVmcb);
6473	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6474	return VERR_SVM_UNEXPECTED_EXIT;
6475	}
6476
6477
6478	/**
6479	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
6480	*/
6481	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6482	{
6483	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6484
6485	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6486	Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6487	#ifdef VBOX_WITH_STATISTICS
6488	switch (pSvmTransient->u64ExitCode)
6489	{
6490	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
6491	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
6492	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
6493	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
6494	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
6495	}
6496	#endif
6497
6498	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
6499	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6500	if ( fSupportsDecodeAssists
6501	&& fSupportsNextRipSave)
6502	{
6503	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6504	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6505	if (fMovCRx)
6506	{
6507	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
6508	\| CPUMCTX_EXTRN_APIC_TPR);
6509	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6510	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
6511	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6512	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6513	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6514	return VBOXSTRICTRC_VAL(rcStrict);
6515	}
6516	/* else: SMSW instruction, fall back below to IEM for this. */
6517	}
6518
6519	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6520	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6521	AssertMsg( rcStrict == VINF_SUCCESS
6522	\|\| rcStrict == VINF_PGM_SYNC_CR3
6523	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
6524	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6525	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
6526	if (rcStrict == VINF_IEM_RAISED_XCPT)
6527	{
6528	rcStrict = VINF_SUCCESS;
6529	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6530	}
6531	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6532	return VBOXSTRICTRC_TODO(rcStrict);
6533	}
6534
6535
6536	/**
6537	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
6538	*/
6539	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6540	{
6541	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6542
6543	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6544	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
6545	Assert(iCrReg <= 15);
6546
6547	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
6548	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6549	bool fDecodedInstr = false;
6550	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
6551	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6552	if ( fSupportsDecodeAssists
6553	&& fSupportsNextRipSave)
6554	{
6555	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6556	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6557	if (fMovCRx)
6558	{
6559	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
6560	\| CPUMCTX_EXTRN_APIC_TPR);
6561	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6562	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6563	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
6564	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6565	fDecodedInstr = true;
6566	}
6567	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
6568	}
6569
6570	if (!fDecodedInstr)
6571	{
6572	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6573	Log4Func(("iCrReg=%#x\n", iCrReg));
6574	rcStrict = IEMExecOne(pVCpu);
6575	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
6576	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
6577	rcStrict = VERR_EM_INTERPRETER;
6578	}
6579
6580	if (rcStrict == VINF_SUCCESS)
6581	{
6582	switch (iCrReg)
6583	{
6584	case 0:
6585	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
6586	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
6587	break;
6588
6589	case 2:
6590	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
6591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
6592	break;
6593
6594	case 3:
6595	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
6596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
6597	break;
6598
6599	case 4:
6600	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
6601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
6602	break;
6603
6604	case 8:
6605	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6606	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
6607	break;
6608
6609	default:
6610	{
6611	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
6612	pSvmTransient->u64ExitCode, iCrReg));
6613	break;
6614	}
6615	}
6616	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6617	}
6618	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6619	{
6620	rcStrict = VINF_SUCCESS;
6621	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6622	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6623	}
6624	else
6625	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
6626	return VBOXSTRICTRC_TODO(rcStrict);
6627	}
6628
6629
6630	/**
6631	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
6632	*
6633	* @returns Strict VBox status code.
6634	* @param pVCpu The cross context virtual CPU structure.
6635	* @param pVmcb Pointer to the VM control block.
6636	*/
6637	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPU pVCpu, PSVMVMCB pVmcb)
6638	{
6639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
6640	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
6641
6642	VBOXSTRICTRC rcStrict;
6643	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6644	if (fSupportsNextRipSave)
6645	{
6646	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
6647	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
6648	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
6649	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6650	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
6651	}
6652	else
6653	{
6654	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
6655	rcStrict = IEMExecOne(pVCpu);
6656	}
6657
6658	AssertMsg( rcStrict == VINF_SUCCESS
6659	\|\| rcStrict == VINF_IEM_RAISED_XCPT
6660	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
6661	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6662
6663	if (rcStrict == VINF_IEM_RAISED_XCPT)
6664	{
6665	rcStrict = VINF_SUCCESS;
6666	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6667	}
6668	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6669	return rcStrict;
6670	}
6671
6672
6673	/**
6674	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
6675	*
6676	* @returns Strict VBox status code.
6677	* @param pVCpu The cross context virtual CPU structure.
6678	* @param pVmcb Pointer to the VM control block.
6679	* @param pSvmTransient Pointer to the SVM-transient structure.
6680	*/
6681	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPU pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
6682	{
6683	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6684	uint32_t const idMsr = pCtx->ecx;
6685	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
6686	Log4Func(("idMsr=%#RX32\n", idMsr));
6687
6688	/*
6689	* Handle TPR patching MSR writes.
6690	* We utilitize the LSTAR MSR for patching.
6691	*/
6692	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6693	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive
6694	&& idMsr == MSR_K8_LSTAR)
6695	{
6696	unsigned cbInstr;
6697	if (fSupportsNextRipSave)
6698	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6699	else
6700	{
6701	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
6702	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
6703	if ( rc == VINF_SUCCESS
6704	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
6705	Assert(cbInstr > 0);
6706	else
6707	cbInstr = 0;
6708	}
6709
6710	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
6711	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
6712	{
6713	int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
6714	AssertRCReturn(rc, rc);
6715	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6716	}
6717
6718	int rc = VINF_SUCCESS;
6719	hmR0SvmAdvanceRip(pVCpu, cbInstr);
6720	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6721	return rc;
6722	}
6723
6724	/*
6725	* Handle regular MSR writes.
6726	*/
6727	VBOXSTRICTRC rcStrict;
6728	if (fSupportsNextRipSave)
6729	{
6730	/** @todo Optimize this: We don't need to get much of the MSR state here
6731	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
6732	* clear the applicable extern flags. */
6733	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
6734	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6735	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
6736	}
6737	else
6738	{
6739	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
6740	rcStrict = IEMExecOne(pVCpu);
6741	}
6742
6743	AssertMsg( rcStrict == VINF_SUCCESS
6744	\|\| rcStrict == VINF_IEM_RAISED_XCPT
6745	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
6746	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6747
6748	if (rcStrict == VINF_SUCCESS)
6749	{
6750	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
6751	if ( idMsr >= MSR_IA32_X2APIC_START
6752	&& idMsr <= MSR_IA32_X2APIC_END)
6753	{
6754	/*
6755	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
6756	* When full APIC register virtualization is implemented we'll have to make sure
6757	* APIC state is saved from the VMCB before IEM changes it.
6758	*/
6759	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6760	}
6761	else
6762	{
6763	switch (idMsr)
6764	{
6765	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
6766	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
6767	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
6768	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
6769	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
6770	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
6771	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
6772	}
6773	}
6774	}
6775	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6776	{
6777	rcStrict = VINF_SUCCESS;
6778	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6779	}
6780	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6781	return rcStrict;
6782	}
6783
6784
6785	/**
6786	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
6787	* \#VMEXIT.
6788	*/
6789	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6790	{
6791	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6792
6793	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6794	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
6795	return VBOXSTRICTRC_TODO(hmR0SvmExitReadMsr(pVCpu, pVmcb));
6796
6797	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
6798	return VBOXSTRICTRC_TODO(hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient));
6799	}
6800
6801
6802	/**
6803	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
6804	*/
6805	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6806	{
6807	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6808	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6809
6810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
6811
6812	/** @todo Stepping with nested-guest. */
6813	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6814	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
6815	{
6816	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
6817	if (pSvmTransient->fWasGuestDebugStateActive)
6818	{
6819	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
6820	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6821	return VERR_SVM_UNEXPECTED_EXIT;
6822	}
6823
6824	/*
6825	* Lazy DR0-3 loading.
6826	*/
6827	if (!pSvmTransient->fWasHyperDebugStateActive)
6828	{
6829	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
6830	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
6831
6832	/* Don't intercept DRx read and writes. */
6833	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6834	pVmcb->ctrl.u16InterceptRdDRx = 0;
6835	pVmcb->ctrl.u16InterceptWrDRx = 0;
6836	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
6837
6838	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6839	VMMRZCallRing3Disable(pVCpu);
6840	HM_DISABLE_PREEMPT(pVCpu);
6841
6842	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
6843	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
6844	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
6845
6846	HM_RESTORE_PREEMPT();
6847	VMMRZCallRing3Enable(pVCpu);
6848
6849	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
6850	return VINF_SUCCESS;
6851	}
6852	}
6853
6854	/*
6855	* Interpret the read/writing of DRx.
6856	*/
6857	/** @todo Decode assist. */
6858	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6859	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
6860	if (RT_LIKELY(rc == VINF_SUCCESS))
6861	{
6862	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
6863	/** @todo CPUM should set this flag! */
6864	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
6865	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6866	}
6867	else
6868	Assert(rc == VERR_EM_INTERPRETER);
6869	return VBOXSTRICTRC_TODO(rc);
6870	}
6871
6872
6873	/**
6874	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
6875	*/
6876	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6877	{
6878	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6879	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
6880	int rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
6881	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
6882	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
6883	return rc;
6884	}
6885
6886
6887	/**
6888	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
6889	*/
6890	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6891	{
6892	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6893	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6894
6895	/** @todo decode assists... */
6896	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6897	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
6898	{
6899	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6900	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6901	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], pVCpu->hm.s.fLoadSaveGuestXcr0,
6902	pCtx->cr4));
6903	}
6904	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6905	{
6906	rcStrict = VINF_SUCCESS;
6907	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6908	}
6909	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6910	return VBOXSTRICTRC_TODO(rcStrict);
6911	}
6912
6913
6914	/**
6915	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
6916	*/
6917	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
6918	{
6919	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6920	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
6921
6922	/* I/O operation lookup arrays. */
6923	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
6924	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
6925	the result (in AL/AX/EAX). */
6926	PVM pVM = pVCpu->CTX_SUFF(pVM);
6927	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6928	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6929
6930	Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6931
6932	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
6933	SVMIOIOEXITINFO IoExitInfo;
6934	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
6935	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
6936	uint32_t cbValue = s_aIOSize[uIOWidth];
6937	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
6938
6939	if (RT_UNLIKELY(!cbValue))
6940	{
6941	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
6942	return VERR_EM_INTERPRETER;
6943	}
6944
6945	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
6946	VBOXSTRICTRC rcStrict;
6947	PCEMEXITREC pExitRec = NULL;
6948	if ( !pVCpu->hm.s.fSingleInstruction
6949	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
6950	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6951	!IoExitInfo.n.u1Str
6952	? IoExitInfo.n.u1Type == SVM_IOIO_READ
6953	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
6954	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
6955	: IoExitInfo.n.u1Type == SVM_IOIO_READ
6956	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
6957	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
6958	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6959	if (!pExitRec)
6960	{
6961	bool fUpdateRipAlready = false;
6962	if (IoExitInfo.n.u1Str)
6963	{
6964	/* INS/OUTS - I/O String instruction. */
6965	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
6966	* in EXITINFO1? Investigate once this thing is up and running. */
6967	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
6968	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
6969	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
6970	static IEMMODE const s_aenmAddrMode[8] =
6971	{
6972	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
6973	};
6974	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
6975	if (enmAddrMode != (IEMMODE)-1)
6976	{
6977	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
6978	if (cbInstr <= 15 && cbInstr >= 1)
6979	{
6980	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
6981	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6982	{
6983	/* Don't know exactly how to detect whether u3Seg is valid, currently
6984	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
6985	2384 Opterons when only checking NRIP. */
6986	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6987	if ( fSupportsNextRipSave
6988	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
6989	{
6990	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
6991	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
6992	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
6993	IoExitInfo.n.u3Seg, true /fIoChecked/);
6994	}
6995	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
6996	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
6997	X86_SREG_DS, true /fIoChecked/);
6998	else
6999	rcStrict = IEMExecOne(pVCpu);
7000	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7001	}
7002	else
7003	{
7004	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7005	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7006	true /fIoChecked/);
7007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7008	}
7009	}
7010	else
7011	{
7012	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7013	rcStrict = IEMExecOne(pVCpu);
7014	}
7015	}
7016	else
7017	{
7018	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7019	rcStrict = IEMExecOne(pVCpu);
7020	}
7021	fUpdateRipAlready = true;
7022	}
7023	else
7024	{
7025	/* IN/OUT - I/O instruction. */
7026	Assert(!IoExitInfo.n.u1Rep);
7027
7028	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7029	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7030	{
7031	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7032	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7033	&& !pCtx->eflags.Bits.u1TF)
7034	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7035	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7036	}
7037	else
7038	{
7039	uint32_t u32Val = 0;
7040	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7041	if (IOM_SUCCESS(rcStrict))
7042	{
7043	/* Save result of I/O IN instr. in AL/AX/EAX. */
7044	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7045	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7046	}
7047	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7048	&& !pCtx->eflags.Bits.u1TF)
7049	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7050
7051	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7052	}
7053	}
7054
7055	if (IOM_SUCCESS(rcStrict))
7056	{
7057	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7058	if (!fUpdateRipAlready)
7059	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7060
7061	/*
7062	* If any I/O breakpoints are armed, we need to check if one triggered
7063	* and take appropriate action.
7064	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7065	*/
7066	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7067	* execution engines about whether hyper BPs and such are pending. */
7068	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7069	uint32_t const uDr7 = pCtx->dr[7];
7070	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7071	&& X86_DR7_ANY_RW_IO(uDr7)
7072	&& (pCtx->cr4 & X86_CR4_DE))
7073	\|\| DBGFBpIsHwIoArmed(pVM)))
7074	{
7075	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7076	VMMRZCallRing3Disable(pVCpu);
7077	HM_DISABLE_PREEMPT(pVCpu);
7078
7079	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7080	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7081
7082	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7083	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7084	{
7085	/* Raise #DB. */
7086	pVmcb->guest.u64DR6 = pCtx->dr[6];
7087	pVmcb->guest.u64DR7 = pCtx->dr[7];
7088	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7089	hmR0SvmSetPendingXcptDB(pVCpu);
7090	}
7091	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7092	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7093	else if ( rcStrict2 != VINF_SUCCESS
7094	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7095	rcStrict = rcStrict2;
7096	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7097
7098	HM_RESTORE_PREEMPT();
7099	VMMRZCallRing3Enable(pVCpu);
7100	}
7101
7102	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7103	}
7104
7105	#ifdef VBOX_STRICT
7106	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7107	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7108	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7109	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7110	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7111	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7112	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7113	else
7114	{
7115	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7116	* statuses, that the VMM device and some others may return. See
7117	* IOM_SUCCESS() for guidance. */
7118	AssertMsg( RT_FAILURE(rcStrict)
7119	\|\| rcStrict == VINF_SUCCESS
7120	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7121	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7122	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7123	\|\| rcStrict == VINF_EM_RAW_TO_R3
7124	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED
7125	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7126	}
7127	#endif
7128	}
7129	else
7130	{
7131	/*
7132	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7133	*/
7134	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7135	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7136	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7137	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7138	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7139	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7140	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7141
7142	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7143	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7144
7145	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7146	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7147	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7148	}
7149	return VBOXSTRICTRC_TODO(rcStrict);
7150	}
7151
7152
7153	/**
7154	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7155	*/
7156	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7157	{
7158	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7159	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7160	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7161
7162	PVM pVM = pVCpu->CTX_SUFF(pVM);
7163	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7164	Assert(pVM->hm.s.fNestedPaging);
7165
7166	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7167	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7168	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7169	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7170	thus intentionally not copied into u32ErrCode. */
7171
7172	Log4Func(("#NPF at CS:RIP=%04x:%#RX64 GCPhysFaultAddr=%RGp ErrCode=%#x \n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7173	u32ErrCode));
7174
7175	/*
7176	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7177	*/
7178	if ( pVM->hm.s.fTprPatchingAllowed
7179	&& (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7180	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7181	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7182	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7183	&& !CPUMIsGuestInLongModeEx(pCtx)
7184	&& !CPUMGetGuestCPL(pVCpu)
7185	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7186	{
7187	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7188	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7189
7190	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7191	{
7192	/* Only attempt to patch the instruction once. */
7193	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7194	if (!pPatch)
7195	return VINF_EM_HM_PATCH_TPR_INSTR;
7196	}
7197	}
7198
7199	/*
7200	* Determine the nested paging mode.
7201	*/
7202	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7203	PGMMODE enmNestedPagingMode;
7204	#if HC_ARCH_BITS == 32
7205	if (CPUMIsGuestInLongModeEx(pCtx))
7206	enmNestedPagingMode = PGMMODE_AMD64_NX;
7207	else
7208	#endif
7209	enmNestedPagingMode = PGMGetHostMode(pVM);
7210
7211	/*
7212	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7213	*/
7214	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7215	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7216	{
7217	/*
7218	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7219	* injecting the original pending event would most likely cause the same MMIO #NPF.
7220	*/
7221	if (pVCpu->hm.s.Event.fPending)
7222	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7223
7224	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7225	VBOXSTRICTRC rcStrict;
7226	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7227	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7228	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7229	if (!pExitRec)
7230	{
7231
7232	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
7233	u32ErrCode);
7234
7235	/*
7236	* If we succeed, resume guest execution.
7237	*
7238	* If we fail in interpreting the instruction because we couldn't get the guest
7239	* physical address of the page containing the instruction via the guest's page
7240	* tables (we would invalidate the guest page in the host TLB), resume execution
7241	* which would cause a guest page fault to let the guest handle this weird case.
7242	*
7243	* See @bugref{6043}.
7244	*/
7245	if ( rcStrict == VINF_SUCCESS
7246	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7247	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
7248	{
7249	/* Successfully handled MMIO operation. */
7250	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7251	rcStrict = VINF_SUCCESS;
7252	}
7253	}
7254	else
7255	{
7256	/*
7257	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7258	*/
7259	Assert(pCtx == &pVCpu->cpum.GstCtx);
7260	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7261	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7262	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
7263
7264	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7265	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7266
7267	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7268	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7269	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7270	}
7271	return VBOXSTRICTRC_TODO(rcStrict);
7272	}
7273
7274	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
7275	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
7276	TRPMResetTrap(pVCpu);
7277
7278	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
7279
7280	/*
7281	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
7282	*/
7283	if ( rc == VINF_SUCCESS
7284	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7285	\|\| rc == VERR_PAGE_NOT_PRESENT)
7286	{
7287	/* We've successfully synced our shadow page tables. */
7288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7289	rc = VINF_SUCCESS;
7290	}
7291
7292	return rc;
7293	}
7294
7295
7296	/**
7297	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
7298	* \#VMEXIT.
7299	*/
7300	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7301	{
7302	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7303	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
7304
7305	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7306	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7307	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
7308
7309	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7310	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7311	return VINF_SUCCESS;
7312	}
7313
7314
7315	/**
7316	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
7317	* \#VMEXIT.
7318	*/
7319	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7320	{
7321	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7322	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7323
7324	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
7325	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
7326	#endif
7327
7328	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
7329	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
7330	{
7331	/*
7332	* AMD-V provides us with the exception which caused the TS; we collect
7333	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
7334	*/
7335	Log4Func(("TS occurred during event delivery\n"));
7336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7337	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7338	}
7339
7340	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7341	* emulation. */
7342	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7343	return VERR_EM_INTERPRETER;
7344	}
7345
7346
7347	/**
7348	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7349	*/
7350	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7351	{
7352	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7353	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7354
7355	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingAllowed)
7356	{
7357	int rc = hmSvmEmulateMovTpr(pVCpu);
7358	if (rc != VERR_NOT_FOUND)
7359	{
7360	Log4Func(("hmSvmEmulateMovTpr returns %Rrc\n", rc));
7361	return rc;
7362	}
7363	}
7364
7365	if (EMAreHypercallInstructionsEnabled(pVCpu))
7366	{
7367	unsigned cbInstr;
7368	if (hmR0SvmSupportsNextRipSave(pVCpu))
7369	{
7370	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7371	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7372	}
7373	else
7374	{
7375	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7376	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7377	if ( rc == VINF_SUCCESS
7378	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
7379	Assert(cbInstr > 0);
7380	else
7381	cbInstr = 0;
7382	}
7383
7384	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
7385	if (RT_SUCCESS(rcStrict))
7386	{
7387	/* Only update the RIP if we're continuing guest execution and not in the case
7388	of say VINF_GIM_R3_HYPERCALL. */
7389	if (rcStrict == VINF_SUCCESS)
7390	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7391
7392	return VBOXSTRICTRC_VAL(rcStrict);
7393	}
7394	else
7395	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
7396	}
7397
7398	hmR0SvmSetPendingXcptUD(pVCpu);
7399	return VINF_SUCCESS;
7400	}
7401
7402
7403	/**
7404	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7405	*/
7406	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7407	{
7408	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7409
7410	unsigned cbInstr;
7411	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7412	if (fSupportsNextRipSave)
7413	{
7414	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7415	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7416	}
7417	else
7418	{
7419	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7420	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7421	if ( rc == VINF_SUCCESS
7422	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
7423	Assert(cbInstr > 0);
7424	else
7425	cbInstr = 0;
7426	}
7427
7428	/** @todo The guest has likely hit a contended spinlock. We might want to
7429	* poke a schedule different guest VCPU. */
7430	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7431	return VINF_EM_RAW_INTERRUPT;
7432	}
7433
7434
7435	/**
7436	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
7437	* \#VMEXIT.
7438	*/
7439	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7440	{
7441	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7442	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
7443	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
7444
7445	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
7446	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
7447	}
7448
7449
7450	/**
7451	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
7452	*/
7453	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7454	{
7455	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7456
7457	/* Clear NMI blocking. */
7458	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7459	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7460
7461	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7462	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7463	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
7464
7465	/* Deliver the pending NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7466	return VINF_SUCCESS;
7467	}
7468
7469
7470	/**
7471	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
7472	* Conditional \#VMEXIT.
7473	*/
7474	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7475	{
7476	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7477	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7478	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7479
7480	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7481	PVM pVM = pVCpu->CTX_SUFF(pVM);
7482	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7483	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7484	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
7485	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7486
7487	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
7488	if (pVM->hm.s.fNestedPaging)
7489	{
7490	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7491	if ( !pSvmTransient->fVectoringDoublePF
7492	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7493	{
7494	/* A genuine guest #PF, reflect it to the guest. */
7495	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
7496	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
7497	uFaultAddress, uErrCode));
7498	}
7499	else
7500	{
7501	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7502	hmR0SvmSetPendingXcptDF(pVCpu);
7503	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
7504	}
7505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7506	return VINF_SUCCESS;
7507	}
7508	#endif
7509
7510	Assert(!pVM->hm.s.fNestedPaging);
7511
7512	/*
7513	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
7514	*/
7515	if ( pVM->hm.s.fTprPatchingAllowed
7516	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
7517	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
7518	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7519	&& !CPUMIsGuestInLongModeEx(pCtx)
7520	&& !CPUMGetGuestCPL(pVCpu)
7521	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7522	{
7523	RTGCPHYS GCPhysApicBase;
7524	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7525	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7526
7527	/* Check if the page at the fault-address is the APIC base. */
7528	RTGCPHYS GCPhysPage;
7529	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
7530	if ( rc2 == VINF_SUCCESS
7531	&& GCPhysPage == GCPhysApicBase)
7532	{
7533	/* Only attempt to patch the instruction once. */
7534	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7535	if (!pPatch)
7536	return VINF_EM_HM_PATCH_TPR_INSTR;
7537	}
7538	}
7539
7540	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7541	pCtx->rip, uErrCode, pCtx->cr3));
7542
7543	/*
7544	* If it's a vectoring #PF, emulate injecting the original event injection as
7545	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
7546	* event injection that caused a #PF. See @bugref{6607}.
7547	*/
7548	if (pSvmTransient->fVectoringPF)
7549	{
7550	Assert(pVCpu->hm.s.Event.fPending);
7551	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7552	}
7553
7554	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
7555	int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7556
7557	Log4Func(("#PF: rc=%Rrc\n", rc));
7558
7559	if (rc == VINF_SUCCESS)
7560	{
7561	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7562	TRPMResetTrap(pVCpu);
7563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7564	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7565	return rc;
7566	}
7567
7568	if (rc == VINF_EM_RAW_GUEST_TRAP)
7569	{
7570	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7571
7572	/*
7573	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
7574	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
7575	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
7576	*/
7577	if ( !pSvmTransient->fVectoringDoublePF
7578	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7579	{
7580	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
7581	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7582	TRPMResetTrap(pVCpu);
7583
7584	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7585	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
7586	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7587	&& HMIsGuestSvmXcptInterceptSet(pVCpu, X86_XCPT_PF))
7588	return VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress));
7589	#endif
7590
7591	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
7592	}
7593	else
7594	{
7595	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7596	TRPMResetTrap(pVCpu);
7597	hmR0SvmSetPendingXcptDF(pVCpu);
7598	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
7599	}
7600
7601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7602	return VINF_SUCCESS;
7603	}
7604
7605	TRPMResetTrap(pVCpu);
7606	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7607	return rc;
7608	}
7609
7610
7611	/**
7612	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
7613	* Conditional \#VMEXIT.
7614	*/
7615	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7616	{
7617	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7618	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
7619
7620	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7621	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7622	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7623
7624	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
7625	if (pVCpu->hm.s.fGIMTrapXcptUD)
7626	{
7627	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7628	uint8_t cbInstr = 0;
7629	VBOXSTRICTRC rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
7630	if (rcStrict == VINF_SUCCESS)
7631	{
7632	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
7633	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7634	rc = VINF_SUCCESS;
7635	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7636	}
7637	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
7638	rc = VINF_SUCCESS;
7639	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
7640	rc = VINF_GIM_R3_HYPERCALL;
7641	else
7642	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
7643	}
7644
7645	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
7646	if (RT_FAILURE(rc))
7647	{
7648	hmR0SvmSetPendingXcptUD(pVCpu);
7649	rc = VINF_SUCCESS;
7650	}
7651
7652	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7653	return rc;
7654	}
7655
7656
7657	/**
7658	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
7659	* Conditional \#VMEXIT.
7660	*/
7661	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7662	{
7663	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7664	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7665
7666	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7667	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7668
7669	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7670	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7671
7672	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
7673
7674	if (!(pCtx->cr0 & X86_CR0_NE))
7675	{
7676	PVM pVM = pVCpu->CTX_SUFF(pVM);
7677	PDISSTATE pDis = &pVCpu->hm.s.DisState;
7678	unsigned cbInstr;
7679	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbInstr);
7680	if (RT_SUCCESS(rc))
7681	{
7682	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
7683	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
7684	if (RT_SUCCESS(rc))
7685	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7686	}
7687	else
7688	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
7689	return rc;
7690	}
7691
7692	hmR0SvmSetPendingXcptMF(pVCpu);
7693	return VINF_SUCCESS;
7694	}
7695
7696
7697	/**
7698	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
7699	* \#VMEXIT.
7700	*/
7701	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7702	{
7703	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7704	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7705	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7706
7707	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7708	{
7709	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7710	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7711	}
7712
7713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
7714
7715	/*
7716	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
7717	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
7718	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
7719	*/
7720	PVM pVM = pVCpu->CTX_SUFF(pVM);
7721	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7722	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7723	int rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
7724	if (rc == VINF_EM_RAW_GUEST_TRAP)
7725	{
7726	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
7727	if (CPUMIsHyperDebugStateActive(pVCpu))
7728	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
7729
7730	/* Reflect the exception back to the guest. */
7731	hmR0SvmSetPendingXcptDB(pVCpu);
7732	rc = VINF_SUCCESS;
7733	}
7734
7735	/*
7736	* Update DR6.
7737	*/
7738	if (CPUMIsHyperDebugStateActive(pVCpu))
7739	{
7740	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
7741	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
7742	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7743	}
7744	else
7745	{
7746	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
7747	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
7748	}
7749
7750	return rc;
7751	}
7752
7753
7754	/**
7755	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
7756	* Conditional \#VMEXIT.
7757	*/
7758	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7759	{
7760	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7761	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7762
7763	SVMEVENT Event;
7764	Event.u = 0;
7765	Event.n.u1Valid = 1;
7766	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7767	Event.n.u8Vector = X86_XCPT_AC;
7768	Event.n.u1ErrorCodeValid = 1;
7769	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7770	return VINF_SUCCESS;
7771	}
7772
7773
7774	/**
7775	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
7776	* Conditional \#VMEXIT.
7777	*/
7778	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7779	{
7780	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7781	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7782	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7783
7784	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7785	int rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7786	if (rc == VINF_EM_RAW_GUEST_TRAP)
7787	{
7788	SVMEVENT Event;
7789	Event.u = 0;
7790	Event.n.u1Valid = 1;
7791	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7792	Event.n.u8Vector = X86_XCPT_BP;
7793	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7794	}
7795
7796	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
7797	return rc;
7798	}
7799
7800
7801	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM)
7802	/**
7803	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
7804	*/
7805	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7806	{
7807	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7808	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7809
7810	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7811	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
7812	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
7813	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
7814	Assert(uVector <= X86_XCPT_LAST);
7815	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
7816
7817	SVMEVENT Event;
7818	Event.u = 0;
7819	Event.n.u1Valid = 1;
7820	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7821	Event.n.u8Vector = uVector;
7822	switch (uVector)
7823	{
7824	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
7825	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
7826	case X86_XCPT_DF:
7827	case X86_XCPT_TS:
7828	case X86_XCPT_NP:
7829	case X86_XCPT_SS:
7830	case X86_XCPT_GP:
7831	case X86_XCPT_AC:
7832	{
7833	Event.n.u1ErrorCodeValid = 1;
7834	Event.n.u32ErrorCode = uErrCode;
7835	break;
7836	}
7837	}
7838
7839	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7840	return VINF_SUCCESS;
7841	}
7842	#endif
7843
7844	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7845	/**
7846	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
7847	*/
7848	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7849	{
7850	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7851
7852	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7853	Assert(pVmcb);
7854	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
7855
7856	VBOXSTRICTRC rcStrict;
7857	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7858	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
7859	if (fSupportsNextRipSave)
7860	{
7861	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
7862	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7863	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
7864	}
7865	else
7866	{
7867	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
7868	rcStrict = IEMExecOne(pVCpu);
7869	}
7870
7871	if (rcStrict == VINF_SUCCESS)
7872	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
7873	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7874	{
7875	rcStrict = VINF_SUCCESS;
7876	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7877	}
7878	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7879	return VBOXSTRICTRC_TODO(rcStrict);
7880	}
7881
7882
7883	/**
7884	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
7885	*/
7886	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7887	{
7888	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7889
7890	/*
7891	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
7892	* we only intercept STGI when events are pending for GIF to become 1.
7893	*/
7894	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7895	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
7896	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
7897
7898	VBOXSTRICTRC rcStrict;
7899	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7900	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
7901	if (fSupportsNextRipSave)
7902	{
7903	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
7904	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7905	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
7906	}
7907	else
7908	{
7909	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
7910	rcStrict = IEMExecOne(pVCpu);
7911	}
7912
7913	if (rcStrict == VINF_SUCCESS)
7914	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
7915	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7916	{
7917	rcStrict = VINF_SUCCESS;
7918	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7919	}
7920	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7921	return VBOXSTRICTRC_TODO(rcStrict);
7922	}
7923
7924
7925	/**
7926	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
7927	*/
7928	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7929	{
7930	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7931
7932	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7933	Assert(pVmcb);
7934	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7935
7936	VBOXSTRICTRC rcStrict;
7937	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7938	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
7939	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
7940	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
7941	if (fSupportsNextRipSave)
7942	{
7943	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
7944	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7945	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
7946	}
7947	else
7948	{
7949	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
7950	rcStrict = IEMExecOne(pVCpu);
7951	}
7952
7953	if (rcStrict == VINF_SUCCESS)
7954	{
7955	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
7956	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
7957	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
7958	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
7959	}
7960	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7961	{
7962	rcStrict = VINF_SUCCESS;
7963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7964	}
7965	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7966	return VBOXSTRICTRC_TODO(rcStrict);
7967	}
7968
7969
7970	/**
7971	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
7972	*/
7973	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
7974	{
7975	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7976
7977	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7978	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7979
7980	VBOXSTRICTRC rcStrict;
7981	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7982	if (fSupportsNextRipSave)
7983	{
7984	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7985	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7986	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
7987	}
7988	else
7989	{
7990	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7991	rcStrict = IEMExecOne(pVCpu);
7992	}
7993
7994	if (rcStrict == VINF_IEM_RAISED_XCPT)
7995	{
7996	rcStrict = VINF_SUCCESS;
7997	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7998	}
7999	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8000	return VBOXSTRICTRC_TODO(rcStrict);
8001	}
8002
8003
8004	/**
8005	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
8006	*/
8007	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
8008	{
8009	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8010
8011	VBOXSTRICTRC rcStrict;
8012	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8013	if (fSupportsNextRipSave)
8014	{
8015	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
8016	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8017	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8018	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
8019	}
8020	else
8021	{
8022	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
8023	rcStrict = IEMExecOne(pVCpu);
8024	}
8025
8026	if (rcStrict == VINF_IEM_RAISED_XCPT)
8027	{
8028	rcStrict = VINF_SUCCESS;
8029	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8030	}
8031	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8032	return VBOXSTRICTRC_TODO(rcStrict);
8033	}
8034
8035
8036	/**
8037	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
8038	*/
8039	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
8040	{
8041	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8042	/* We shall import the entire state here, just in case we enter and continue execution of
8043	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
8044	could lose lose part of CPU state. */
8045	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8046
8047	VBOXSTRICTRC rcStrict;
8048	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8049	if (fSupportsNextRipSave)
8050	{
8051	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8052	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8053	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
8054	}
8055	else
8056	{
8057	/* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
8058	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
8059	instruction itself, see @bugref{7243#c126} */
8060	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), NULL /* pcbWritten */);
8061	}
8062
8063	if (rcStrict == VINF_SUCCESS)
8064	{
8065	rcStrict = VINF_SVM_VMRUN;
8066	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
8067	}
8068	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8069	{
8070	rcStrict = VINF_SUCCESS;
8071	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8072	}
8073	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8074	return VBOXSTRICTRC_TODO(rcStrict);
8075	}
8076
8077
8078	/**
8079	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
8080	* Unconditional \#VMEXIT.
8081	*/
8082	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
8083	{
8084	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8085	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8086
8087	if (pVCpu->hm.s.Event.fPending)
8088	{
8089	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
8090	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8091	}
8092
8093	hmR0SvmSetPendingXcptDB(pVCpu);
8094	return VINF_SUCCESS;
8095	}
8096
8097
8098	/**
8099	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8100	* Conditional \#VMEXIT.
8101	*/
8102	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPU pVCpu, PSVMTRANSIENT pSvmTransient)
8103	{
8104	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8105	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8106
8107	SVMEVENT Event;
8108	Event.u = 0;
8109	Event.n.u1Valid = 1;
8110	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8111	Event.n.u8Vector = X86_XCPT_BP;
8112	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8113	return VINF_SUCCESS;
8114	}
8115	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
8116
8117	/** @} */
8118

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 75649

Download in other formats: