HMSVMR0.cpp@ 93368

Last change on this file since 93368 was 93119, checked in by vboxsync, 3 years ago
VMM/HMSVMR0: Must deal with VINF_IEM_RAISED_XCPT in the string I/O case of an IOIO exit. Debug builds would guru if the guest was using invalid memory and IEM was forced to raise a #PF or #GP. Reordered fCtxChanged/rcStrict updating in similar code paths, placing the rcStrict after the fCtxChange since that's closer to the return point and might help with register liveness.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 379.8 KB

Line
1	/* $Id: HMSVMR0.cpp 93119 2022-01-04 02:11:01Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#include "HMInternal.h"
36	#include <VBox/vmm/vmcc.h>
37	#include <VBox/err.h>
38	#include "HMSVMR0.h"
39	#include "dtrace/VBoxVMM.h"
40
41	#ifdef DEBUG_ramshankar
42	# define HMSVM_SYNC_FULL_GUEST_STATE
43	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
44	# define HMSVM_ALWAYS_TRAP_PF
45	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
46	#endif
47
48
49	/*********************************************************************************************************************************
50	* Defined Constants And Macros *
51	*********************************************************************************************************************************/
52	#ifdef VBOX_WITH_STATISTICS
53	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
54	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
55	if ((u64ExitCode) == SVM_EXIT_NPF) \
56	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
57	else \
58	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
59	} while (0)
60
61	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
62	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
63	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
64	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
65	if ((u64ExitCode) == SVM_EXIT_NPF) \
66	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
67	else \
68	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
69	} while (0)
70	# endif
71	#else
72	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
73	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
74	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
75	# endif
76	#endif /* !VBOX_WITH_STATISTICS */
77
78	/** If we decide to use a function table approach this can be useful to
79	* switch to a "static DECLCALLBACK(int)". */
80	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
81
82	/**
83	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
84	* guest using hardware-assisted SVM.
85	*
86	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
87	* are swapped and restored across the world-switch and also registers like
88	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
89	* \#VMEXIT.
90	*/
91	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
92	\| CPUMCTX_EXTRN_RFLAGS \
93	\| CPUMCTX_EXTRN_RAX \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_CR0 \
97	\| CPUMCTX_EXTRN_CR2 \
98	\| CPUMCTX_EXTRN_CR3 \
99	\| CPUMCTX_EXTRN_TABLE_MASK \
100	\| CPUMCTX_EXTRN_DR6 \
101	\| CPUMCTX_EXTRN_DR7 \
102	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
103	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
104	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
105	\| CPUMCTX_EXTRN_HWVIRT \
106	\| CPUMCTX_EXTRN_INHIBIT_INT \
107	\| CPUMCTX_EXTRN_HM_SVM_MASK)
108
109	/**
110	* Subset of the guest-CPU state that is shared between the guest and host.
111	*/
112	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
113
114	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
115	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
116	do { \
117	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
118	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
119	} while (0)
120
121	/** Assert that the required state bits are fetched. */
122	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
123	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
124	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
125
126	/** Assert that preemption is disabled or covered by thread-context hooks. */
127	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
128	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
129
130	/** Assert that we haven't migrated CPUs when thread-context hooks are not
131	* used. */
132	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
133	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
134	("Illegal migration! Entered on CPU %u Current %u\n", \
135	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
136
137	/** Assert that we're not executing a nested-guest. */
138	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
139	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
140	#else
141	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
142	#endif
143
144	/** Assert that we're executing a nested-guest. */
145	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
146	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
147	#else
148	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
149	#endif
150
151	/** Macro for checking and returning from the using function for
152	* \#VMEXIT intercepts that maybe caused during delivering of another
153	* event in the guest. */
154	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
155	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
156	do \
157	{ \
158	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
159	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
160	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
161	else if ( rc == VINF_EM_RESET \
162	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
163	{ \
164	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
165	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
166	} \
167	else \
168	return rc; \
169	} while (0)
170	#else
171	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
172	do \
173	{ \
174	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
175	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
176	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
177	else \
178	return rc; \
179	} while (0)
180	#endif
181
182	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
183	* instruction that exited. */
184	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
185	do { \
186	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
187	(a_rc) = VINF_EM_DBG_STEPPED; \
188	} while (0)
189
190	/** Validate segment descriptor granularity bit. */
191	#ifdef VBOX_STRICT
192	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
193	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
194	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
195	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
196	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
197	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
198	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
199	#else
200	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
201	#endif
202
203	/**
204	* Exception bitmap mask for all contributory exceptions.
205	*
206	* Page fault is deliberately excluded here as it's conditional as to whether
207	* it's contributory or benign. Page faults are handled separately.
208	*/
209	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
210	\| RT_BIT(X86_XCPT_DE))
211
212	/**
213	* Mandatory/unconditional guest control intercepts.
214	*
215	* SMIs can and do happen in normal operation. We need not intercept them
216	* while executing the guest (or nested-guest).
217	*/
218	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
219	\| SVM_CTRL_INTERCEPT_NMI \
220	\| SVM_CTRL_INTERCEPT_INIT \
221	\| SVM_CTRL_INTERCEPT_RDPMC \
222	\| SVM_CTRL_INTERCEPT_CPUID \
223	\| SVM_CTRL_INTERCEPT_RSM \
224	\| SVM_CTRL_INTERCEPT_HLT \
225	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
226	\| SVM_CTRL_INTERCEPT_MSR_PROT \
227	\| SVM_CTRL_INTERCEPT_INVLPGA \
228	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
229	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
230	\| SVM_CTRL_INTERCEPT_VMRUN \
231	\| SVM_CTRL_INTERCEPT_SKINIT \
232	\| SVM_CTRL_INTERCEPT_WBINVD \
233	\| SVM_CTRL_INTERCEPT_MONITOR \
234	\| SVM_CTRL_INTERCEPT_MWAIT \
235	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
236	\| SVM_CTRL_INTERCEPT_XSETBV)
237
238	/** @name VMCB Clean Bits.
239	*
240	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
241	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
242	* memory.
243	*
244	* @{ */
245	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
246	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
247	/** I/O permission bitmap, MSR permission bitmap. */
248	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
249	/** ASID. */
250	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
251	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
252	V_INTR_VECTOR. */
253	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
254	/** Nested Paging: Nested CR3 (nCR3), PAT. */
255	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
256	/** Control registers (CR0, CR3, CR4, EFER). */
257	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
258	/** Debug registers (DR6, DR7). */
259	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
260	/** GDT, IDT limit and base. */
261	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
262	/** Segment register: CS, SS, DS, ES limit and base. */
263	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
264	/** CR2.*/
265	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
266	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
267	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
268	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
269	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
270	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
271	/** Mask of all valid VMCB Clean bits. */
272	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
273	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
274	\| HMSVM_VMCB_CLEAN_ASID \
275	\| HMSVM_VMCB_CLEAN_INT_CTRL \
276	\| HMSVM_VMCB_CLEAN_NP \
277	\| HMSVM_VMCB_CLEAN_CRX_EFER \
278	\| HMSVM_VMCB_CLEAN_DRX \
279	\| HMSVM_VMCB_CLEAN_DT \
280	\| HMSVM_VMCB_CLEAN_SEG \
281	\| HMSVM_VMCB_CLEAN_CR2 \
282	\| HMSVM_VMCB_CLEAN_LBR \
283	\| HMSVM_VMCB_CLEAN_AVIC)
284	/** @} */
285
286	/** @name SVM transient.
287	*
288	* A state structure for holding miscellaneous information across AMD-V
289	* VMRUN/\#VMEXIT operation, restored after the transition.
290	*
291	* @{ */
292	typedef struct SVMTRANSIENT
293	{
294	/** The host's rflags/eflags. */
295	RTCCUINTREG fEFlags;
296	/** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
297	uint64_t u64ExitCode;
298
299	/** The guest's TPR value used for TPR shadowing. */
300	uint8_t u8GuestTpr;
301	/** Alignment. */
302	uint8_t abAlignment0[7];
303
304	/** Pointer to the currently executing VMCB. */
305	PSVMVMCB pVmcb;
306
307	/** Whether we are currently executing a nested-guest. */
308	bool fIsNestedGuest;
309	/** Whether the guest debug state was active at the time of \#VMEXIT. */
310	bool fWasGuestDebugStateActive;
311	/** Whether the hyper debug state was active at the time of \#VMEXIT. */
312	bool fWasHyperDebugStateActive;
313	/** Whether the TSC offset mode needs to be updated. */
314	bool fUpdateTscOffsetting;
315	/** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
316	bool fRestoreTscAuxMsr;
317	/** Whether the \#VMEXIT was caused by a page-fault during delivery of a
318	* contributary exception or a page-fault. */
319	bool fVectoringDoublePF;
320	/** Whether the \#VMEXIT was caused by a page-fault during delivery of an
321	* external interrupt or NMI. */
322	bool fVectoringPF;
323	/** Padding. */
324	bool afPadding0;
325	} SVMTRANSIENT;
326	/** Pointer to SVM transient state. */
327	typedef SVMTRANSIENT *PSVMTRANSIENT;
328	/** Pointer to a const SVM transient state. */
329	typedef const SVMTRANSIENT *PCSVMTRANSIENT;
330
331	AssertCompileSizeAlignment(SVMTRANSIENT, sizeof(uint64_t));
332	AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
333	AssertCompileMemberAlignment(SVMTRANSIENT, pVmcb, sizeof(uint64_t));
334	/** @} */
335
336	/**
337	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
338	*/
339	typedef enum SVMMSREXITREAD
340	{
341	/** Reading this MSR causes a \#VMEXIT. */
342	SVMMSREXIT_INTERCEPT_READ = 0xb,
343	/** Reading this MSR does not cause a \#VMEXIT. */
344	SVMMSREXIT_PASSTHRU_READ
345	} SVMMSREXITREAD;
346
347	/**
348	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
349	*/
350	typedef enum SVMMSREXITWRITE
351	{
352	/** Writing to this MSR causes a \#VMEXIT. */
353	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
354	/** Writing to this MSR does not cause a \#VMEXIT. */
355	SVMMSREXIT_PASSTHRU_WRITE
356	} SVMMSREXITWRITE;
357
358	/**
359	* SVM \#VMEXIT handler.
360	*
361	* @returns Strict VBox status code.
362	* @param pVCpu The cross context virtual CPU structure.
363	* @param pSvmTransient Pointer to the SVM-transient structure.
364	*/
365	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
366
367
368	/*********************************************************************************************************************************
369	* Internal Functions *
370	*********************************************************************************************************************************/
371	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
372	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
373
374
375	/** @name \#VMEXIT handlers.
376	* @{
377	*/
378	static FNSVMEXITHANDLER hmR0SvmExitIntr;
379	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
380	static FNSVMEXITHANDLER hmR0SvmExitInvd;
381	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
382	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
383	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
384	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
385	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
386	static FNSVMEXITHANDLER hmR0SvmExitHlt;
387	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
388	static FNSVMEXITHANDLER hmR0SvmExitMwait;
389	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
390	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
391	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
392	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
393	static FNSVMEXITHANDLER hmR0SvmExitMsr;
394	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
395	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
396	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
397	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
398	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
399	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
400	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
401	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
402	static FNSVMEXITHANDLER hmR0SvmExitPause;
403	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
404	static FNSVMEXITHANDLER hmR0SvmExitIret;
405	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
406	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
407	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
408	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
409	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
410	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
411	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
412	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
413	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
414	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
415	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
416	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
417	static FNSVMEXITHANDLER hmR0SvmExitClgi;
418	static FNSVMEXITHANDLER hmR0SvmExitStgi;
419	static FNSVMEXITHANDLER hmR0SvmExitVmload;
420	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
421	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
422	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
423	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
424	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
425	#endif
426	/** @} */
427
428	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
429	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
430	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
431	#endif
432	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
433
434
435	/*********************************************************************************************************************************
436	* Global Variables *
437	*********************************************************************************************************************************/
438	/** Ring-0 memory object for the IO bitmap. */
439	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
440	/** Physical address of the IO bitmap. */
441	static RTHCPHYS g_HCPhysIOBitmap;
442	/** Pointer to the IO bitmap. */
443	static R0PTRTYPE(void *) g_pvIOBitmap;
444
445	#ifdef VBOX_STRICT
446	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
447	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
448	# define HMSVM_LOG_CS RT_BIT_32(2)
449	# define HMSVM_LOG_SS RT_BIT_32(3)
450	# define HMSVM_LOG_FS RT_BIT_32(4)
451	# define HMSVM_LOG_GS RT_BIT_32(5)
452	# define HMSVM_LOG_LBR RT_BIT_32(6)
453	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
454	\| HMSVM_LOG_CR_REGS \
455	\| HMSVM_LOG_CS \
456	\| HMSVM_LOG_SS \
457	\| HMSVM_LOG_FS \
458	\| HMSVM_LOG_GS \
459	\| HMSVM_LOG_LBR)
460
461	/**
462	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
463	*
464	* @param pVCpu The cross context virtual CPU structure.
465	* @param pVmcb Pointer to the VM control block.
466	* @param pszPrefix Log prefix.
467	* @param fFlags Log flags, see HMSVM_LOG_XXX.
468	* @param uVerbose The verbosity level, currently unused.
469	*/
470	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
471	{
472	RT_NOREF2(pVCpu, uVerbose);
473	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
474
475	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
476	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
477
478	if (fFlags & HMSVM_LOG_RBP_RSP)
479	{
480	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
481	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
482	}
483
484	if (fFlags & HMSVM_LOG_CR_REGS)
485	{
486	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
487	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
488	}
489
490	if (fFlags & HMSVM_LOG_CS)
491	{
492	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
493	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
494	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
495	}
496	if (fFlags & HMSVM_LOG_SS)
497	{
498	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
499	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
500	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
501	}
502	if (fFlags & HMSVM_LOG_FS)
503	{
504	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
505	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
506	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
507	}
508	if (fFlags & HMSVM_LOG_GS)
509	{
510	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
511	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
512	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
513	}
514
515	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
516	if (fFlags & HMSVM_LOG_LBR)
517	{
518	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
519	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
520	}
521	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
522	}
523	#endif /* VBOX_STRICT */
524
525
526	/**
527	* Sets up and activates AMD-V on the current CPU.
528	*
529	* @returns VBox status code.
530	* @param pHostCpu The HM physical-CPU structure.
531	* @param pVM The cross context VM structure. Can be
532	* NULL after a resume!
533	* @param pvCpuPage Pointer to the global CPU page.
534	* @param HCPhysCpuPage Physical address of the global CPU page.
535	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
536	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
537	* unused).
538	*/
539	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
540	PCSUPHWVIRTMSRS pHwvirtMsrs)
541	{
542	Assert(!fEnabledByHost);
543	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
544	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
545	Assert(pvCpuPage); NOREF(pvCpuPage);
546	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
547
548	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
549
550	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
551	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
552
553	/*
554	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
555	*/
556	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
557	if (u64HostEfer & MSR_K6_EFER_SVME)
558	{
559	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
560	if ( pVM
561	&& pVM->hm.s.svm.fIgnoreInUseError)
562	pHostCpu->fIgnoreAMDVInUseError = true;
563
564	if (!pHostCpu->fIgnoreAMDVInUseError)
565	{
566	ASMSetFlags(fEFlags);
567	return VERR_SVM_IN_USE;
568	}
569	}
570
571	/* Turn on AMD-V in the EFER MSR. */
572	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
573
574	/* Write the physical page address where the CPU will store the host state while executing the VM. */
575	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
576
577	/* Restore interrupts. */
578	ASMSetFlags(fEFlags);
579
580	/*
581	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
582	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
583	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
584	* entirely with before executing any guest code.
585	*/
586	pHostCpu->fFlushAsidBeforeUse = true;
587
588	/*
589	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
590	*/
591	++pHostCpu->cTlbFlushes;
592
593	return VINF_SUCCESS;
594	}
595
596
597	/**
598	* Deactivates AMD-V on the current CPU.
599	*
600	* @returns VBox status code.
601	* @param pHostCpu The HM physical-CPU structure.
602	* @param pvCpuPage Pointer to the global CPU page.
603	* @param HCPhysCpuPage Physical address of the global CPU page.
604	*/
605	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
606	{
607	RT_NOREF1(pHostCpu);
608	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
609	AssertReturn( HCPhysCpuPage
610	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
611	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
612
613	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
614	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
615
616	/* Turn off AMD-V in the EFER MSR. */
617	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
618	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
619
620	/* Invalidate host state physical address. */
621	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
622
623	/* Restore interrupts. */
624	ASMSetFlags(fEFlags);
625
626	return VINF_SUCCESS;
627	}
628
629
630	/**
631	* Does global AMD-V initialization (called during module initialization).
632	*
633	* @returns VBox status code.
634	*/
635	VMMR0DECL(int) SVMR0GlobalInit(void)
636	{
637	/*
638	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
639	* intercept all IO accesses, it's done once globally here instead of per-VM.
640	*/
641	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
642	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
643	if (RT_FAILURE(rc))
644	return rc;
645
646	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
647	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
648
649	/* Set all bits to intercept all IO accesses. */
650	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
651
652	return VINF_SUCCESS;
653	}
654
655
656	/**
657	* Does global AMD-V termination (called during module termination).
658	*/
659	VMMR0DECL(void) SVMR0GlobalTerm(void)
660	{
661	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
662	{
663	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
664	g_pvIOBitmap = NULL;
665	g_HCPhysIOBitmap = 0;
666	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
667	}
668	}
669
670
671	/**
672	* Frees any allocated per-VCPU structures for a VM.
673	*
674	* @param pVM The cross context VM structure.
675	*/
676	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
677	{
678	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
679	{
680	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
681	AssertPtr(pVCpu);
682
683	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
684	{
685	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
686	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
687	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
688	}
689
690	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
691	{
692	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
693	pVCpu->hmr0.s.svm.pVmcb = NULL;
694	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
695	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
696	}
697
698	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
699	{
700	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
701	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
702	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
703	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
704	}
705	}
706	}
707
708
709	/**
710	* Sets pfnVMRun to the best suited variant.
711	*
712	* This must be called whenever anything changes relative to the SVMR0VMRun
713	* variant selection:
714	* - pVCpu->hm.s.fLoadSaveGuestXcr0
715	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
716	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
717	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
718	* - Perhaps: CPUMCTX.fXStateMask (windows only)
719	*
720	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
721	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
722	*/
723	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
724	{
725	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
726	{
727	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
728	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
729	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
730	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
731	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
732	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
733	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
734	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
735	};
736	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
737	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
738	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
739	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
740	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
741	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
742	}
743
744
745	/**
746	* Selector FNHMSVMVMRUN implementation.
747	*/
748	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
749	{
750	hmR0SvmUpdateVmRunFunction(pVCpu);
751	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
752	}
753
754
755	/**
756	* Does per-VM AMD-V initialization.
757	*
758	* @returns VBox status code.
759	* @param pVM The cross context VM structure.
760	*/
761	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
762	{
763	int rc = VERR_INTERNAL_ERROR_5;
764
765	/*
766	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
767	*/
768	uint32_t u32Family;
769	uint32_t u32Model;
770	uint32_t u32Stepping;
771	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
772	{
773	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
774	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
775	}
776
777	/*
778	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
779	*/
780	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
781	{
782	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
783	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
784	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
785	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
786	}
787
788	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
789	{
790	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
791
792	/*
793	* Initialize the hardware-assisted SVM guest-execution handler.
794	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
795	*/
796	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
797
798	/*
799	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
800	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
801	*/
802	/** @todo Does this need to be below 4G? */
803	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
804	if (RT_FAILURE(rc))
805	goto failure_cleanup;
806
807	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
808	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
809	Assert(pVCpu->hmr0.s.svm.HCPhysVmcbHost < _4G);
810	ASMMemZeroPage(pvVmcbHost);
811
812	/*
813	* Allocate one page for the guest-state VMCB.
814	*/
815	/** @todo Does this need to be below 4G? */
816	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
817	if (RT_FAILURE(rc))
818	goto failure_cleanup;
819
820	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
821	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
822	Assert(pVCpu->hmr0.s.svm.HCPhysVmcb < _4G);
823	ASMMemZeroPage(pVCpu->hmr0.s.svm.pVmcb);
824
825	/*
826	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
827	* SVM to not require one.
828	*/
829	/** @todo Does this need to be below 4G? */
830	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
831	false /* fExecutable */);
832	if (RT_FAILURE(rc))
833	goto failure_cleanup;
834
835	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
836	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
837	/* Set all bits to intercept all MSR accesses (changed later on). */
838	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
839	}
840
841	return VINF_SUCCESS;
842
843	failure_cleanup:
844	hmR0SvmFreeStructs(pVM);
845	return rc;
846	}
847
848
849	/**
850	* Does per-VM AMD-V termination.
851	*
852	* @returns VBox status code.
853	* @param pVM The cross context VM structure.
854	*/
855	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
856	{
857	hmR0SvmFreeStructs(pVM);
858	return VINF_SUCCESS;
859	}
860
861
862	/**
863	* Returns whether the VMCB Clean Bits feature is supported.
864	*
865	* @returns @c true if supported, @c false otherwise.
866	* @param pVCpu The cross context virtual CPU structure.
867	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
868	*/
869	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
870	{
871	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
872	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
873	if (!fIsNestedGuest)
874	return fHostVmcbCleanBits;
875	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
876	}
877
878
879	/**
880	* Returns whether the decode assists feature is supported.
881	*
882	* @returns @c true if supported, @c false otherwise.
883	* @param pVCpu The cross context virtual CPU structure.
884	*/
885	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
886	{
887	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
888	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
889	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
890	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
891	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
892	#endif
893	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
894	}
895
896
897	/**
898	* Returns whether the NRIP_SAVE feature is supported.
899	*
900	* @returns @c true if supported, @c false otherwise.
901	* @param pVCpu The cross context virtual CPU structure.
902	*/
903	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
904	{
905	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
906	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
907	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
908	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
909	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
910	#endif
911	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
912	}
913
914
915	/**
916	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
917	*
918	* @param pVCpu The cross context virtual CPU structure.
919	* @param pbMsrBitmap Pointer to the MSR bitmap.
920	* @param idMsr The MSR for which the permissions are being set.
921	* @param enmRead MSR read permissions.
922	* @param enmWrite MSR write permissions.
923	*
924	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
925	* caller needs to take care of this.
926	*/
927	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
928	SVMMSREXITWRITE enmWrite)
929	{
930	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
931	uint16_t offMsrpm;
932	uint8_t uMsrpmBit;
933	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
934	AssertRC(rc);
935
936	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
937	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
938
939	pbMsrBitmap += offMsrpm;
940	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
941	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
942	else
943	{
944	if (!fInNestedGuestMode)
945	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
946	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
947	else
948	{
949	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
950	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
951	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
952	else
953	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
954	}
955	#endif
956	}
957
958	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
959	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
960	else
961	{
962	if (!fInNestedGuestMode)
963	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
964	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
965	else
966	{
967	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
968	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
969	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
970	else
971	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
972	}
973	#endif
974	}
975	}
976
977
978	/**
979	* Sets up AMD-V for the specified VM.
980	* This function is only called once per-VM during initalization.
981	*
982	* @returns VBox status code.
983	* @param pVM The cross context VM structure.
984	*/
985	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
986	{
987	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
988	AssertReturn(pVM, VERR_INVALID_PARAMETER);
989
990	/*
991	* Validate and copy over some parameters.
992	*/
993	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
994	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
995	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
996	pVM->hmr0.s.fNestedPaging = fNestedPaging;
997	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
998
999	/*
1000	* Determin some configuration parameters.
1001	*/
1002	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
1003	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
1004	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
1005
1006	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
1007	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
1008
1009	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1010	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
1011	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
1012
1013	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
1014	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
1015	#endif
1016
1017	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
1018	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
1019	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
1020	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
1021
1022	/* Always trap #AC for reasons of security. */
1023	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
1024
1025	/* Always trap #DB for reasons of security. */
1026	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
1027
1028	/* Trap exceptions unconditionally (debug purposes). */
1029	#ifdef HMSVM_ALWAYS_TRAP_PF
1030	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
1031	#endif
1032	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1033	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1034	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
1035	\| RT_BIT_32(X86_XCPT_DE)
1036	\| RT_BIT_32(X86_XCPT_NM)
1037	\| RT_BIT_32(X86_XCPT_UD)
1038	\| RT_BIT_32(X86_XCPT_NP)
1039	\| RT_BIT_32(X86_XCPT_SS)
1040	\| RT_BIT_32(X86_XCPT_GP)
1041	\| RT_BIT_32(X86_XCPT_PF)
1042	\| RT_BIT_32(X86_XCPT_MF)
1043	;
1044	#endif
1045
1046	/* Apply the exceptions intercepts needed by the GIM provider. */
1047	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1048	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1049
1050	/* The mesa 3d driver hack needs #GP. */
1051	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1052	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1053
1054	/* Set up unconditional intercepts and conditions. */
1055	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1056	\| SVM_CTRL_INTERCEPT_VMMCALL
1057	\| SVM_CTRL_INTERCEPT_VMSAVE
1058	\| SVM_CTRL_INTERCEPT_VMLOAD
1059	\| SVM_CTRL_INTERCEPT_CLGI
1060	\| SVM_CTRL_INTERCEPT_STGI;
1061
1062	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1063	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1064	#endif
1065
1066	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1067	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1068	{
1069	/* Virtualized VMSAVE/VMLOAD. */
1070	if (fUseVirtVmsaveVmload)
1071	{
1072	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1073	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1074	\| SVM_CTRL_INTERCEPT_VMLOAD);
1075	}
1076	else
1077	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1078
1079	/* Virtual GIF. */
1080	if (fUseVGif)
1081	{
1082	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1083	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1084	\| SVM_CTRL_INTERCEPT_STGI);
1085	}
1086	else
1087	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1088	}
1089	else
1090	#endif
1091	{
1092	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1093	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1094	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1095	}
1096
1097	/* CR4 writes must always be intercepted for tracking PGM mode changes. */
1098	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1099
1100	/* Intercept all DRx reads and writes by default. Changed later on. */
1101	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1102	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1103
1104	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1105	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1106
1107	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1108	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1109	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1110
1111	/* Set the IO permission bitmap physical addresses. */
1112	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1113
1114	/* LBR virtualization. */
1115	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1116
1117	/* The host ASID MBZ, for the guest start with 1. */
1118	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1119
1120	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1121	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1122
1123	/* Without Nested Paging, we need additionally intercepts. */
1124	if (!fNestedPaging)
1125	{
1126	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1127	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1128	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1129
1130	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1131	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1132	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1133
1134	/* Page faults must be intercepted to implement shadow paging. */
1135	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1136	}
1137
1138	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1139	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1140	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1141
1142	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1143	if (fUsePauseFilter)
1144	{
1145	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1146	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1147	if (fPauseFilterThreshold)
1148	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1149	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1150	}
1151
1152	/*
1153	* Setup the MSR permission bitmap.
1154	* The following MSRs are saved/restored automatically during the world-switch.
1155	* Don't intercept guest read/write accesses to these MSRs.
1156	*/
1157	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1158	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1159	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1160	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1161	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1162	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1163	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1164	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1165	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1166	{
1167	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1168	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1169	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1170	}
1171	else
1172	{
1173	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1174	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1175	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1176	}
1177	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1178
1179	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1180	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1181
1182	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1183	{
1184	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1185	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1186	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1187	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1188
1189	/* Copy the VMCB control area. */
1190	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1191
1192	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1193	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1194	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1195	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1196
1197	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1198	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1199
1200	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1201	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1202	}
1203
1204	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1205	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1206	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1207	#else
1208	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1209	#endif
1210	return VINF_SUCCESS;
1211	}
1212
1213
1214	/**
1215	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1216	*
1217	* @returns Pointer to the current context VMCB.
1218	* @param pVCpu The cross context virtual CPU structure.
1219	*/
1220	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1221	{
1222	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1223	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1224	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1225	#endif
1226	return pVCpu->hmr0.s.svm.pVmcb;
1227	}
1228
1229
1230	/**
1231	* Gets a pointer to the nested-guest VMCB cache.
1232	*
1233	* @returns Pointer to the nested-guest VMCB cache.
1234	* @param pVCpu The cross context virtual CPU structure.
1235	*/
1236	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1237	{
1238	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1239	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1240	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1241	#else
1242	RT_NOREF(pVCpu);
1243	return NULL;
1244	#endif
1245	}
1246
1247
1248	/**
1249	* Invalidates a guest page by guest virtual address.
1250	*
1251	* @returns VBox status code.
1252	* @param pVCpu The cross context virtual CPU structure.
1253	* @param GCVirt Guest virtual address of the page to invalidate.
1254	*/
1255	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1256	{
1257	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1258
1259	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1260
1261	/* Skip it if a TLB flush is already pending. */
1262	if (!fFlushPending)
1263	{
1264	Log4Func(("%#RGv\n", GCVirt));
1265
1266	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1267	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1268
1269	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1270	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1271	}
1272	return VINF_SUCCESS;
1273	}
1274
1275
1276	/**
1277	* Flushes the appropriate tagged-TLB entries.
1278	*
1279	* @param pHostCpu The HM physical-CPU structure.
1280	* @param pVCpu The cross context virtual CPU structure.
1281	* @param pVmcb Pointer to the VM control block.
1282	*/
1283	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1284	{
1285	/*
1286	* Force a TLB flush for the first world switch if the current CPU differs from the one
1287	* we ran on last. This can happen both for start & resume due to long jumps back to
1288	* ring-3.
1289	*
1290	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1291	* correlation between it and the physical CPU.
1292	*
1293	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1294	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1295	*/
1296	bool fNewAsid = false;
1297	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1298	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1299	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1300	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1301	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1302	#endif
1303	)
1304	{
1305	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1306	pVCpu->hmr0.s.fForceTLBFlush = true;
1307	fNewAsid = true;
1308	}
1309
1310	/* Set TLB flush state as checked until we return from the world switch. */
1311	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1312
1313	/* Check for explicit TLB flushes. */
1314	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1315	{
1316	pVCpu->hmr0.s.fForceTLBFlush = true;
1317	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1318	}
1319
1320	/*
1321	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1322	* This Host CPU requirement takes precedence.
1323	*/
1324	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1325	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1326	{
1327	pHostCpu->uCurrentAsid = 1;
1328	pVCpu->hmr0.s.uCurrentAsid = 1;
1329	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1330	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1331	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1332
1333	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1334	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1335	}
1336	else
1337	{
1338	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1339	if (pVCpu->hmr0.s.fForceTLBFlush)
1340	{
1341	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1342	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1343
1344	if (fNewAsid)
1345	{
1346	++pHostCpu->uCurrentAsid;
1347
1348	bool fHitASIDLimit = false;
1349	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1350	{
1351	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1352	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1353	fHitASIDLimit = true;
1354	}
1355
1356	if ( fHitASIDLimit
1357	\|\| pHostCpu->fFlushAsidBeforeUse)
1358	{
1359	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1360	pHostCpu->fFlushAsidBeforeUse = false;
1361	}
1362
1363	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1364	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1365	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1366	}
1367	else
1368	{
1369	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1370	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1371	else
1372	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1373	}
1374
1375	pVCpu->hmr0.s.fForceTLBFlush = false;
1376	}
1377	}
1378
1379	/* Update VMCB with the ASID. */
1380	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1381	{
1382	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1383	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1384	}
1385
1386	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1387	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1388	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1389	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1390	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1391	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1392	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1393	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1394
1395	#ifdef VBOX_WITH_STATISTICS
1396	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1397	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1398	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1399	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1400	{
1401	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1402	}
1403	else
1404	{
1405	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1406	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1407	}
1408	#endif
1409	}
1410
1411
1412	/**
1413	* Sets an exception intercept in the specified VMCB.
1414	*
1415	* @param pVmcb Pointer to the VM control block.
1416	* @param uXcpt The exception (X86_XCPT_*).
1417	*/
1418	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1419	{
1420	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1421	{
1422	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1423	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1424	}
1425	}
1426
1427
1428	/**
1429	* Clears an exception intercept in the specified VMCB.
1430	*
1431	* @param pVCpu The cross context virtual CPU structure.
1432	* @param pVmcb Pointer to the VM control block.
1433	* @param uXcpt The exception (X86_XCPT_*).
1434	*
1435	* @remarks This takes into account if we're executing a nested-guest and only
1436	* removes the exception intercept if both the guest -and- nested-guest
1437	* are not intercepting it.
1438	*/
1439	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1440	{
1441	Assert(uXcpt != X86_XCPT_DB);
1442	Assert(uXcpt != X86_XCPT_AC);
1443	Assert(uXcpt != X86_XCPT_GP);
1444	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1445	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1446	{
1447	bool fRemove = true;
1448	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1449	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1450	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1451	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1452	{
1453	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1454	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1455	}
1456	# else
1457	RT_NOREF(pVCpu);
1458	# endif
1459	if (fRemove)
1460	{
1461	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1462	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1463	}
1464	}
1465	#else
1466	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1467	#endif
1468	}
1469
1470
1471	/**
1472	* Sets a control intercept in the specified VMCB.
1473	*
1474	* @param pVmcb Pointer to the VM control block.
1475	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1476	*/
1477	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1478	{
1479	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1480	{
1481	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1482	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1483	}
1484	}
1485
1486
1487	/**
1488	* Clears a control intercept in the specified VMCB.
1489	*
1490	* @returns @c true if the intercept is still set, @c false otherwise.
1491	* @param pVCpu The cross context virtual CPU structure.
1492	* @param pVmcb Pointer to the VM control block.
1493	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1494	*
1495	* @remarks This takes into account if we're executing a nested-guest and only
1496	* removes the control intercept if both the guest -and- nested-guest
1497	* are not intercepting it.
1498	*/
1499	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1500	{
1501	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1502	{
1503	bool fRemove = true;
1504	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1505	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1506	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1507	{
1508	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1509	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1510	}
1511	#else
1512	RT_NOREF(pVCpu);
1513	#endif
1514	if (fRemove)
1515	{
1516	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1517	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1518	}
1519	}
1520
1521	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1522	}
1523
1524
1525	/**
1526	* Exports the guest (or nested-guest) CR0 into the VMCB.
1527	*
1528	* @param pVCpu The cross context virtual CPU structure.
1529	* @param pVmcb Pointer to the VM control block.
1530	*
1531	* @remarks This assumes we always pre-load the guest FPU.
1532	* @remarks No-long-jump zone!!!
1533	*/
1534	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1535	{
1536	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537
1538	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1539	uint64_t const uGuestCr0 = pCtx->cr0;
1540	uint64_t uShadowCr0 = uGuestCr0;
1541
1542	/* Always enable caching. */
1543	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1544
1545	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1546	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1547	{
1548	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1549	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1550	}
1551
1552	/*
1553	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1554	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1555	* see @bugref{7243#c103}.
1556	*/
1557	if (!(uGuestCr0 & X86_CR0_NE))
1558	{
1559	uShadowCr0 \|= X86_CR0_NE;
1560	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1561	}
1562	else
1563	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1564
1565	/*
1566	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1567	*
1568	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1569	* see @bugref{6944}.
1570	*
1571	* We also don't ever want to honor weird things like cache disable from the guest.
1572	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1573	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1574	*/
1575	if (uShadowCr0 == uGuestCr0)
1576	{
1577	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1578	{
1579	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1580	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1581	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1582	}
1583	else
1584	{
1585	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1586	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1587	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1588	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1589	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1590	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1591	}
1592	}
1593	else
1594	{
1595	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1596	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1597	}
1598	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1599
1600	Assert(!RT_HI_U32(uShadowCr0));
1601	if (pVmcb->guest.u64CR0 != uShadowCr0)
1602	{
1603	pVmcb->guest.u64CR0 = uShadowCr0;
1604	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1605	}
1606	}
1607
1608
1609	/**
1610	* Exports the guest (or nested-guest) CR3 into the VMCB.
1611	*
1612	* @param pVCpu The cross context virtual CPU structure.
1613	* @param pVmcb Pointer to the VM control block.
1614	*
1615	* @remarks No-long-jump zone!!!
1616	*/
1617	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1618	{
1619	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1620
1621	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1622	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1623	if (pVM->hmr0.s.fNestedPaging)
1624	{
1625	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1626	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1627	pVmcb->guest.u64CR3 = pCtx->cr3;
1628	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1629	}
1630	else
1631	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1632
1633	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1634	}
1635
1636
1637	/**
1638	* Exports the guest (or nested-guest) CR4 into the VMCB.
1639	*
1640	* @param pVCpu The cross context virtual CPU structure.
1641	* @param pVmcb Pointer to the VM control block.
1642	*
1643	* @remarks No-long-jump zone!!!
1644	*/
1645	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1646	{
1647	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1648
1649	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1650	uint64_t uShadowCr4 = pCtx->cr4;
1651	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1652	{
1653	switch (pVCpu->hm.s.enmShadowMode)
1654	{
1655	case PGMMODE_REAL:
1656	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1657	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1658
1659	case PGMMODE_32_BIT: /* 32-bit paging. */
1660	uShadowCr4 &= ~X86_CR4_PAE;
1661	break;
1662
1663	case PGMMODE_PAE: /* PAE paging. */
1664	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1665	/** Must use PAE paging as we could use physical memory > 4 GB */
1666	uShadowCr4 \|= X86_CR4_PAE;
1667	break;
1668
1669	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1670	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1671	#ifdef VBOX_WITH_64_BITS_GUESTS
1672	break;
1673	#else
1674	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1675	#endif
1676
1677	default: /* shut up gcc */
1678	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1679	}
1680	}
1681
1682	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1683	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1684	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1685	{
1686	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1687	hmR0SvmUpdateVmRunFunction(pVCpu);
1688	}
1689
1690	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1691	if (uShadowCr4 == pCtx->cr4)
1692	{
1693	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1694	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1695	else
1696	{
1697	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1698	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1699	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1700	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1701	}
1702	}
1703	else
1704	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1705
1706	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking PGM mode changes. */
1707	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1708
1709	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1710	Assert(!RT_HI_U32(uShadowCr4));
1711	pVmcb->guest.u64CR4 = uShadowCr4;
1712	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1713
1714	return VINF_SUCCESS;
1715	}
1716
1717
1718	/**
1719	* Exports the guest (or nested-guest) control registers into the VMCB.
1720	*
1721	* @returns VBox status code.
1722	* @param pVCpu The cross context virtual CPU structure.
1723	* @param pVmcb Pointer to the VM control block.
1724	*
1725	* @remarks No-long-jump zone!!!
1726	*/
1727	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1728	{
1729	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1730
1731	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1732	{
1733	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1734	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1735
1736	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1737	{
1738	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1739	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1740	}
1741
1742	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1743	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1744
1745	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1746	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1747	{
1748	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1749	if (RT_FAILURE(rc))
1750	return rc;
1751	}
1752
1753	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1754	}
1755	return VINF_SUCCESS;
1756	}
1757
1758
1759	/**
1760	* Exports the guest (or nested-guest) segment registers into the VMCB.
1761	*
1762	* @returns VBox status code.
1763	* @param pVCpu The cross context virtual CPU structure.
1764	* @param pVmcb Pointer to the VM control block.
1765	*
1766	* @remarks No-long-jump zone!!!
1767	*/
1768	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1769	{
1770	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1771	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1772
1773	/* Guest segment registers. */
1774	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1775	{
1776	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1777	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1778
1779	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1780	{
1781	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1782	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1783	}
1784
1785	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1786	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1787
1788	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1789	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1790
1791	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1792	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1793
1794	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1795	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1796
1797	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1798	}
1799
1800	/* Guest TR. */
1801	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1802	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1803
1804	/* Guest LDTR. */
1805	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1806	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1807
1808	/* Guest GDTR. */
1809	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1810	{
1811	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1812	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1813	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1814	}
1815
1816	/* Guest IDTR. */
1817	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1818	{
1819	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1820	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1821	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1822	}
1823
1824	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1825	\| HM_CHANGED_GUEST_TABLE_MASK);
1826	}
1827
1828
1829	/**
1830	* Exports the guest (or nested-guest) MSRs into the VMCB.
1831	*
1832	* @param pVCpu The cross context virtual CPU structure.
1833	* @param pVmcb Pointer to the VM control block.
1834	*
1835	* @remarks No-long-jump zone!!!
1836	*/
1837	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1838	{
1839	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1840	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1841
1842	/* Guest Sysenter MSRs. */
1843	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1844	{
1845	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1846	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1847
1848	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1849	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1850
1851	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1852	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1853	}
1854
1855	/*
1856	* Guest EFER MSR.
1857	* AMD-V requires guest EFER.SVME to be set. Weird.
1858	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1859	*/
1860	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1861	{
1862	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1863	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1864	}
1865
1866	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1867	if ( !CPUMIsGuestInLongModeEx(pCtx)
1868	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1869	{
1870	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1871	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1872	}
1873
1874	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1875	{
1876	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1877	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1878	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1879	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1880	}
1881
1882	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1883	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1884
1885	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1886	\| HM_CHANGED_GUEST_EFER_MSR
1887	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1888	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1889
1890	/*
1891	* Setup the PAT MSR (applicable for Nested Paging only).
1892	*
1893	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1894	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1895	*
1896	* While guests can modify and see the modified values through the shadow values,
1897	* we shall not honor any guest modifications of this MSR to ensure caching is always
1898	* enabled similar to how we clear CR0.CD and NW bits.
1899	*
1900	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1901	*/
1902	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1903
1904	/* Enable the last branch record bit if LBR virtualization is enabled. */
1905	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1906	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1907	}
1908
1909
1910	/**
1911	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1912	* the necessary intercepts accordingly.
1913	*
1914	* @param pVCpu The cross context virtual CPU structure.
1915	* @param pVmcb Pointer to the VM control block.
1916	*
1917	* @remarks No-long-jump zone!!!
1918	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1919	*/
1920	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1921	{
1922	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1923
1924	/** @todo Figure out stepping with nested-guest. */
1925	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1926	{
1927	/*
1928	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1929	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1930	* Instead, they are strictly only requested when the nested hypervisor intercepts
1931	* them -- handled while merging VMCB controls.
1932	*
1933	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1934	* then the nested-guest debug state should be actively loaded on the host so that
1935	* nested-guest reads/writes its own debug registers without causing VM-exits.
1936	*/
1937	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1938	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1939	&& !CPUMIsGuestDebugStateActive(pVCpu))
1940	{
1941	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1942	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1943	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1944	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1945	}
1946
1947	pVmcb->guest.u64DR6 = pCtx->dr[6];
1948	pVmcb->guest.u64DR7 = pCtx->dr[7];
1949	return;
1950	}
1951
1952	/*
1953	* Anyone single stepping on the host side? If so, we'll have to use the
1954	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1955	* the VMM level like the VT-x implementations does.
1956	*/
1957	bool fInterceptMovDRx = false;
1958	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1959	if (fStepping)
1960	{
1961	pVCpu->hmr0.s.fClearTrapFlag = true;
1962	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1963	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1964	}
1965
1966	if ( fStepping
1967	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1968	{
1969	/*
1970	* Use the combined guest and host DRx values found in the hypervisor
1971	* register set because the debugger has breakpoints active or someone
1972	* is single stepping on the host side.
1973	*
1974	* Note! DBGF expects a clean DR6 state before executing guest code.
1975	*/
1976	if (!CPUMIsHyperDebugStateActive(pVCpu))
1977	{
1978	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1979	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1980	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1981	}
1982
1983	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1984	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1985	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1986	{
1987	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1988	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1989	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1990	}
1991
1992	/** @todo If we cared, we could optimize to allow the guest to read registers
1993	* with the same values. */
1994	fInterceptMovDRx = true;
1995	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1996	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1997	}
1998	else
1999	{
2000	/*
2001	* Update DR6, DR7 with the guest values if necessary.
2002	*/
2003	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
2004	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
2005	{
2006	pVmcb->guest.u64DR7 = pCtx->dr[7];
2007	pVmcb->guest.u64DR6 = pCtx->dr[6];
2008	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
2009	}
2010	pVCpu->hmr0.s.fUsingHyperDR7 = false;
2011
2012	/*
2013	* If the guest has enabled debug registers, we need to load them prior to
2014	* executing guest code so they'll trigger at the right time.
2015	*/
2016	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
2017	{
2018	if (!CPUMIsGuestDebugStateActive(pVCpu))
2019	{
2020	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
2021	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2022	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2023	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2024	}
2025	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2026	}
2027	/*
2028	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2029	* intercept #DB as DR6 is updated in the VMCB.
2030	*
2031	* Note! If we cared and dared, we could skip intercepting \#DB here.
2032	* However, \#DB shouldn't be performance critical, so we'll play safe
2033	* and keep the code similar to the VT-x code and always intercept it.
2034	*/
2035	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2036	fInterceptMovDRx = true;
2037	}
2038
2039	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2040	if (fInterceptMovDRx)
2041	{
2042	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2043	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2044	{
2045	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2046	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2047	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2048	}
2049	}
2050	else
2051	{
2052	if ( pVmcb->ctrl.u16InterceptRdDRx
2053	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2054	{
2055	pVmcb->ctrl.u16InterceptRdDRx = 0;
2056	pVmcb->ctrl.u16InterceptWrDRx = 0;
2057	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2058	}
2059	}
2060	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2061	}
2062
2063	/**
2064	* Exports the hardware virtualization state into the nested-guest
2065	* VMCB.
2066	*
2067	* @param pVCpu The cross context virtual CPU structure.
2068	* @param pVmcb Pointer to the VM control block.
2069	*
2070	* @remarks No-long-jump zone!!!
2071	*/
2072	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2073	{
2074	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2075
2076	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2077	{
2078	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2079	{
2080	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2081	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2082
2083	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2084	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2085	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2086	NOREF(pVM);
2087
2088	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2089	}
2090
2091	/*
2092	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2093	* since SVM doesn't have a preemption timer.
2094	*
2095	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2096	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2097	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2098	*/
2099	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2100	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2101	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2102	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2103	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2104	{
2105	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2106	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2107	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2108	}
2109	else
2110	{
2111	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2112	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2113	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2114	}
2115	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2116
2117	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2118	}
2119	}
2120
2121
2122	/**
2123	* Exports the guest APIC TPR state into the VMCB.
2124	*
2125	* @returns VBox status code.
2126	* @param pVCpu The cross context virtual CPU structure.
2127	* @param pVmcb Pointer to the VM control block.
2128	*/
2129	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2130	{
2131	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2132
2133	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2134	{
2135	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2136	if ( PDMHasApic(pVM)
2137	&& APICIsEnabled(pVCpu))
2138	{
2139	bool fPendingIntr;
2140	uint8_t u8Tpr;
2141	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2142	AssertRCReturn(rc, rc);
2143
2144	/* Assume that we need to trap all TPR accesses and thus need not check on
2145	every #VMEXIT if we should update the TPR. */
2146	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2147	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2148
2149	if (!pVM->hm.s.fTprPatchingActive)
2150	{
2151	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2152	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2153
2154	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2155	can deliver the interrupt to the guest. */
2156	if (fPendingIntr)
2157	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2158	else
2159	{
2160	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2161	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2162	}
2163
2164	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2165	}
2166	else
2167	{
2168	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2169	pVmcb->guest.u64LSTAR = u8Tpr;
2170	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2171
2172	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2173	if (fPendingIntr)
2174	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2175	else
2176	{
2177	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2178	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2179	}
2180	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2181	}
2182	}
2183	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2184	}
2185	return VINF_SUCCESS;
2186	}
2187
2188
2189	/**
2190	* Sets up the exception interrupts required for guest execution in the VMCB.
2191	*
2192	* @param pVCpu The cross context virtual CPU structure.
2193	* @param pVmcb Pointer to the VM control block.
2194	*
2195	* @remarks No-long-jump zone!!!
2196	*/
2197	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2198	{
2199	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2200
2201	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2202	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2203	{
2204	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2205	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2206	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2207	else
2208	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2209
2210	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2211	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2212	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2213	else
2214	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2215
2216	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2217	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2218	}
2219	}
2220
2221
2222	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2223	/**
2224	* Merges guest and nested-guest intercepts for executing the nested-guest using
2225	* hardware-assisted SVM.
2226	*
2227	* This merges the guest and nested-guest intercepts in a way that if the outer
2228	* guest intercept is set we need to intercept it in the nested-guest as
2229	* well.
2230	*
2231	* @param pVCpu The cross context virtual CPU structure.
2232	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2233	*/
2234	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2235	{
2236	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2237	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2238	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2239	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2240
2241	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2242	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2243	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2244
2245	/* Always intercept CR4 writes for tracking PGM mode changes. */
2246	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2247
2248	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2249	if (!pVM->hmr0.s.fNestedPaging)
2250	{
2251	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2252	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2253	}
2254
2255	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2256	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2257	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2258
2259	/*
2260	* Merge the guest's exception intercepts into the nested-guest VMCB.
2261	*
2262	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2263	* while executing the nested-guest.
2264	*
2265	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2266	* be tweaked later depending on how we wish to implement breakpoints.
2267	*
2268	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2269	* loaded into their guests, not ours.
2270	*
2271	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2272	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2273	*/
2274	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2275	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2276	& ~( RT_BIT(X86_XCPT_UD)
2277	\| RT_BIT(X86_XCPT_BP)
2278	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2279	#else
2280	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2281	#endif
2282
2283	/*
2284	* Adjust intercepts while executing the nested-guest that differ from the
2285	* outer guest intercepts.
2286	*
2287	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2288	* that belong to the nested-guest to the outer guest.
2289	*
2290	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2291	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2292	*/
2293	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2294	\| SVM_CTRL_INTERCEPT_VMMCALL))
2295	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2296
2297	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2298	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2299
2300	/* Finally, update the VMCB clean bits. */
2301	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2302	}
2303	#endif
2304
2305
2306	/**
2307	* Enters the AMD-V session.
2308	*
2309	* @returns VBox status code.
2310	* @param pVCpu The cross context virtual CPU structure.
2311	*/
2312	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2313	{
2314	AssertPtr(pVCpu);
2315	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2316	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2317
2318	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2319	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2320	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2321
2322	pVCpu->hmr0.s.fLeaveDone = false;
2323	return VINF_SUCCESS;
2324	}
2325
2326
2327	/**
2328	* Thread-context callback for AMD-V.
2329	*
2330	* This is used together with RTThreadCtxHookCreate() on platforms which
2331	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2332	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2333	*
2334	* @param enmEvent The thread-context event.
2335	* @param pVCpu The cross context virtual CPU structure.
2336	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2337	* @thread EMT(pVCpu)
2338	*/
2339	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2340	{
2341	NOREF(fGlobalInit);
2342
2343	switch (enmEvent)
2344	{
2345	case RTTHREADCTXEVENT_OUT:
2346	{
2347	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2348	VMCPU_ASSERT_EMT(pVCpu);
2349
2350	/* No longjmps (log-flush, locks) in this fragile context. */
2351	VMMRZCallRing3Disable(pVCpu);
2352
2353	if (!pVCpu->hmr0.s.fLeaveDone)
2354	{
2355	hmR0SvmLeave(pVCpu, false /* fImportState */);
2356	pVCpu->hmr0.s.fLeaveDone = true;
2357	}
2358
2359	/* Leave HM context, takes care of local init (term). */
2360	int rc = HMR0LeaveCpu(pVCpu);
2361	AssertRC(rc); NOREF(rc);
2362
2363	/* Restore longjmp state. */
2364	VMMRZCallRing3Enable(pVCpu);
2365	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2366	break;
2367	}
2368
2369	case RTTHREADCTXEVENT_IN:
2370	{
2371	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2372	VMCPU_ASSERT_EMT(pVCpu);
2373
2374	/* No longjmps (log-flush, locks) in this fragile context. */
2375	VMMRZCallRing3Disable(pVCpu);
2376
2377	/*
2378	* Initialize the bare minimum state required for HM. This takes care of
2379	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2380	*/
2381	int rc = hmR0EnterCpu(pVCpu);
2382	AssertRC(rc); NOREF(rc);
2383	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2384	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2385
2386	pVCpu->hmr0.s.fLeaveDone = false;
2387
2388	/* Restore longjmp state. */
2389	VMMRZCallRing3Enable(pVCpu);
2390	break;
2391	}
2392
2393	default:
2394	break;
2395	}
2396	}
2397
2398
2399	/**
2400	* Saves the host state.
2401	*
2402	* @returns VBox status code.
2403	* @param pVCpu The cross context virtual CPU structure.
2404	*
2405	* @remarks No-long-jump zone!!!
2406	*/
2407	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2408	{
2409	NOREF(pVCpu);
2410
2411	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2412	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2413	return VINF_SUCCESS;
2414	}
2415
2416
2417	/**
2418	* Exports the guest or nested-guest state from the virtual-CPU context into the
2419	* VMCB.
2420	*
2421	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2422	* code based on the virtual-CPU mode.
2423	*
2424	* @returns VBox status code.
2425	* @param pVCpu The cross context virtual CPU structure.
2426	* @param pSvmTransient Pointer to the SVM-transient structure.
2427	*
2428	* @remarks No-long-jump zone!!!
2429	*/
2430	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2431	{
2432	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2433
2434	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2435	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2436	Assert(pVmcb);
2437
2438	pVmcb->guest.u64RIP = pCtx->rip;
2439	pVmcb->guest.u64RSP = pCtx->rsp;
2440	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2441	pVmcb->guest.u64RAX = pCtx->rax;
2442
2443	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2444	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2445
2446	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2447	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2448	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2449	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2450	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2451
2452	ASMSetFlags(fEFlags);
2453
2454	if (!fIsNestedGuest)
2455	{
2456	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2457	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2458	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2459	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2460	}
2461
2462	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2463	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2464	\| HM_CHANGED_GUEST_RFLAGS
2465	\| HM_CHANGED_GUEST_GPRS_MASK
2466	\| HM_CHANGED_GUEST_X87
2467	\| HM_CHANGED_GUEST_SSE_AVX
2468	\| HM_CHANGED_GUEST_OTHER_XSAVE
2469	\| HM_CHANGED_GUEST_XCRx
2470	\| HM_CHANGED_GUEST_TSC_AUX
2471	\| HM_CHANGED_GUEST_OTHER_MSRS;
2472	if (fIsNestedGuest)
2473	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2474	\| HM_CHANGED_GUEST_APIC_TPR;
2475
2476	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2477	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2478
2479	#ifdef VBOX_STRICT
2480	/*
2481	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2482	* except for the host-context and/or shared host-guest context bits.
2483	*/
2484	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2485	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2486	("fCtxChanged=%#RX64\n", fCtxChanged));
2487
2488	/*
2489	* If we need to log state that isn't always imported, we'll need to import them here.
2490	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2491	*/
2492	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2493	#endif
2494
2495	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2496	return VINF_SUCCESS;
2497	}
2498
2499	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2500
2501	/**
2502	* Merges the guest and nested-guest MSR permission bitmap.
2503	*
2504	* If the guest is intercepting an MSR we need to intercept it regardless of
2505	* whether the nested-guest is intercepting it or not.
2506	*
2507	* @param pHostCpu The HM physical-CPU structure.
2508	* @param pVCpu The cross context virtual CPU structure.
2509	*
2510	* @remarks No-long-jmp zone!!!
2511	*/
2512	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2513	{
2514	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2515	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2516	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2517
2518	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2519	uint32_t const offRsvdQwords = 0x1800 >> 3;
2520	for (uint32_t i = 0; i < offRsvdQwords; i++)
2521	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2522	}
2523
2524
2525	/**
2526	* Caches the nested-guest VMCB fields before we modify them for execution using
2527	* hardware-assisted SVM.
2528	*
2529	* @returns true if the VMCB was previously already cached, false otherwise.
2530	* @param pVCpu The cross context virtual CPU structure.
2531	*
2532	* @sa HMNotifySvmNstGstVmexit.
2533	*/
2534	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2535	{
2536	/*
2537	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2538	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2539	*
2540	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2541	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2542	*/
2543	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2544	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2545	if (!fWasCached)
2546	{
2547	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2548	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2549	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2550	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2551	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2552	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2553	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2554	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2555	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2556	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2557	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2558	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2559	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2560	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2561	pVmcbNstGstCache->fCacheValid = true;
2562	Log4Func(("Cached VMCB fields\n"));
2563	}
2564
2565	return fWasCached;
2566	}
2567
2568
2569	/**
2570	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2571	*
2572	* This is done the first time we enter nested-guest execution using SVM R0
2573	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2574	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2575	*
2576	* @param pVCpu The cross context virtual CPU structure.
2577	*/
2578	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2579	{
2580	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2581	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2582
2583	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2584
2585	/*
2586	* First cache the nested-guest VMCB fields we may potentially modify.
2587	*/
2588	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2589	if (!fVmcbCached)
2590	{
2591	/*
2592	* The IOPM of the nested-guest can be ignored because the the guest always
2593	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2594	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2595	*/
2596	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2597
2598	/*
2599	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2600	* nested-paging suddenly while executing a VM (see assertion at the end of
2601	* Trap0eHandler() in PGMAllBth.h).
2602	*/
2603	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2604
2605	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2606	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2607
2608	/*
2609	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2610	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2611	*/
2612	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2613
2614	# ifdef DEBUG_ramshankar
2615	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2616	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2617	# endif
2618
2619	/*
2620	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2621	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2622	*/
2623	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2624	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2625	\| SVM_CTRL_INTERCEPT_VMLOAD;
2626
2627	/*
2628	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2629	* CLGI/STGI instructions executed by the nested-guest.
2630	*/
2631	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2632	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2633	\| SVM_CTRL_INTERCEPT_STGI;
2634
2635	/* Merge the guest and nested-guest intercepts. */
2636	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2637
2638	/* Update the VMCB clean bits. */
2639	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2640	}
2641	else
2642	{
2643	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2644	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2645	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2646	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2647	}
2648	}
2649
2650	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2651
2652	/**
2653	* Exports the state shared between the host and guest (or nested-guest) into
2654	* the VMCB.
2655	*
2656	* @param pVCpu The cross context virtual CPU structure.
2657	* @param pVmcb Pointer to the VM control block.
2658	*
2659	* @remarks No-long-jump zone!!!
2660	*/
2661	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2662	{
2663	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2664	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2665
2666	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2667	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2668
2669	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2670	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2671	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2672	}
2673
2674
2675	/**
2676	* Worker for SVMR0ImportStateOnDemand.
2677	*
2678	* @param pVCpu The cross context virtual CPU structure.
2679	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2680	*/
2681	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2682	{
2683	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2684
2685	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2686	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2687	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2688	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2689
2690	/*
2691	* We disable interrupts to make the updating of the state and in particular
2692	* the fExtrn modification atomic wrt to preemption hooks.
2693	*/
2694	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2695
2696	fWhat &= pCtx->fExtrn;
2697	if (fWhat)
2698	{
2699	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2700	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2701	{
2702	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2703	{
2704	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2705	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2706	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2707	}
2708	}
2709
2710	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2711	{
2712	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2713	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2714	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2715	}
2716	#endif
2717
2718	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2719	{
2720	if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2721	EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
2722	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2723	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2724	}
2725
2726	if (fWhat & CPUMCTX_EXTRN_RIP)
2727	pCtx->rip = pVmcbGuest->u64RIP;
2728
2729	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2730	pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
2731
2732	if (fWhat & CPUMCTX_EXTRN_RSP)
2733	pCtx->rsp = pVmcbGuest->u64RSP;
2734
2735	if (fWhat & CPUMCTX_EXTRN_RAX)
2736	pCtx->rax = pVmcbGuest->u64RAX;
2737
2738	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2739	{
2740	if (fWhat & CPUMCTX_EXTRN_CS)
2741	{
2742	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2743	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2744	/** @todo SELM might need to be fixed as it too should not care about the
2745	* granularity bit. See @bugref{6785}. */
2746	if ( !pCtx->cs.Attr.n.u1Granularity
2747	&& pCtx->cs.Attr.n.u1Present
2748	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2749	{
2750	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2751	pCtx->cs.Attr.n.u1Granularity = 1;
2752	}
2753	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2754	}
2755	if (fWhat & CPUMCTX_EXTRN_SS)
2756	{
2757	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2758	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2759	/*
2760	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2761	* VMCB and uses that and thus it's possible that when the CPL changes during
2762	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2763	* AMD Fusion CPUs with 64-bit guests.
2764	*
2765	* See AMD spec. 15.5.1 "Basic operation".
2766	*/
2767	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2768	uint8_t const uCpl = pVmcbGuest->u8CPL;
2769	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2770	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2771	}
2772	if (fWhat & CPUMCTX_EXTRN_DS)
2773	{
2774	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2775	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2776	}
2777	if (fWhat & CPUMCTX_EXTRN_ES)
2778	{
2779	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2780	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2781	}
2782	if (fWhat & CPUMCTX_EXTRN_FS)
2783	{
2784	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2785	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2786	}
2787	if (fWhat & CPUMCTX_EXTRN_GS)
2788	{
2789	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2790	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2791	}
2792	}
2793
2794	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2795	{
2796	if (fWhat & CPUMCTX_EXTRN_TR)
2797	{
2798	/*
2799	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2800	* are used between Intel and AMD, see @bugref{6208#c39}.
2801	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2802	*/
2803	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2804	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2805	{
2806	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2807	\|\| CPUMIsGuestInLongModeEx(pCtx))
2808	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2809	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2810	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2811	}
2812	}
2813
2814	if (fWhat & CPUMCTX_EXTRN_LDTR)
2815	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2816
2817	if (fWhat & CPUMCTX_EXTRN_GDTR)
2818	{
2819	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2820	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2821	}
2822
2823	if (fWhat & CPUMCTX_EXTRN_IDTR)
2824	{
2825	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2826	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2827	}
2828	}
2829
2830	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2831	{
2832	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2833	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2834	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2835	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2836	}
2837
2838	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2839	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2840	{
2841	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2842	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2843	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2844	}
2845
2846	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2847	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2848
2849	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2850	{
2851	if (fWhat & CPUMCTX_EXTRN_DR6)
2852	{
2853	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2854	pCtx->dr[6] = pVmcbGuest->u64DR6;
2855	else
2856	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2857	}
2858
2859	if (fWhat & CPUMCTX_EXTRN_DR7)
2860	{
2861	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2862	pCtx->dr[7] = pVmcbGuest->u64DR7;
2863	else
2864	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2865	}
2866	}
2867
2868	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2869	{
2870	if (fWhat & CPUMCTX_EXTRN_CR0)
2871	{
2872	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2873	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2874	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2875	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2876	CPUMSetGuestCR0(pVCpu, uCr0);
2877	VMMRZCallRing3Enable(pVCpu);
2878	}
2879
2880	if (fWhat & CPUMCTX_EXTRN_CR2)
2881	pCtx->cr2 = pVmcbGuest->u64CR2;
2882
2883	if (fWhat & CPUMCTX_EXTRN_CR3)
2884	{
2885	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2886	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2887	{
2888	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2889	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2890	}
2891	}
2892
2893	/* Changes to CR4 are always intercepted. */
2894	}
2895
2896	/* Update fExtrn. */
2897	pCtx->fExtrn &= ~fWhat;
2898
2899	/* If everything has been imported, clear the HM keeper bit. */
2900	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2901	{
2902	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2903	Assert(!pCtx->fExtrn);
2904	}
2905	}
2906	else
2907	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2908
2909	ASMSetFlags(fEFlags);
2910
2911	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2912
2913	/*
2914	* Honor any pending CR3 updates.
2915	*
2916	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2917	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2918	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2919	* handling -> hmR0SvmImportGuestState() and here we are.
2920	*
2921	* The reason for such complicated handling is because VM-exits that call into PGM expect
2922	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2923	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2924	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2925	*
2926	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2927	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2928	*/
2929	if ( VMMRZCallRing3IsEnabled(pVCpu)
2930	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2931	{
2932	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2933	PGMUpdateCR3(pVCpu, pCtx->cr3);
2934	}
2935	}
2936
2937
2938	/**
2939	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2940	* context.
2941	*
2942	* Currently there is no residual state left in the CPU that is not updated in the
2943	* VMCB.
2944	*
2945	* @returns VBox status code.
2946	* @param pVCpu The cross context virtual CPU structure.
2947	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2948	*/
2949	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2950	{
2951	hmR0SvmImportGuestState(pVCpu, fWhat);
2952	return VINF_SUCCESS;
2953	}
2954
2955
2956	/**
2957	* Does the necessary state syncing before returning to ring-3 for any reason
2958	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2959	*
2960	* @param pVCpu The cross context virtual CPU structure.
2961	* @param fImportState Whether to import the guest state from the VMCB back
2962	* to the guest-CPU context.
2963	*
2964	* @remarks No-long-jmp zone!!!
2965	*/
2966	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2967	{
2968	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2969	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2970
2971	/*
2972	* !!! IMPORTANT !!!
2973	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2974	*/
2975
2976	/* Save the guest state if necessary. */
2977	if (fImportState)
2978	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2979
2980	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2981	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2982	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2983
2984	/*
2985	* Restore host debug registers if necessary and resync on next R0 reentry.
2986	*/
2987	#ifdef VBOX_STRICT
2988	if (CPUMIsHyperDebugStateActive(pVCpu))
2989	{
2990	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2991	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2992	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2993	}
2994	#endif
2995	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2996	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2997	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2998
2999	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3000	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3001	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3002	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3004	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3005	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3006
3007	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3008	}
3009
3010
3011	/**
3012	* Leaves the AMD-V session.
3013	*
3014	* Only used while returning to ring-3 either due to longjump or exits to
3015	* ring-3.
3016	*
3017	* @returns VBox status code.
3018	* @param pVCpu The cross context virtual CPU structure.
3019	*/
3020	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3021	{
3022	HM_DISABLE_PREEMPT(pVCpu);
3023	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3024	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3025
3026	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3027	and done this from the SVMR0ThreadCtxCallback(). */
3028	if (!pVCpu->hmr0.s.fLeaveDone)
3029	{
3030	hmR0SvmLeave(pVCpu, true /* fImportState */);
3031	pVCpu->hmr0.s.fLeaveDone = true;
3032	}
3033
3034	/*
3035	* !!! IMPORTANT !!!
3036	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3037	*/
3038
3039	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3040	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3041	VMMR0ThreadCtxHookDisable(pVCpu);
3042
3043	/* Leave HM context. This takes care of local init (term). */
3044	int rc = HMR0LeaveCpu(pVCpu);
3045
3046	HM_RESTORE_PREEMPT();
3047	return rc;
3048	}
3049
3050
3051	/**
3052	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3053	* any remaining host state) before we go back to ring-3 due to an assertion.
3054	*
3055	* @param pVCpu The cross context virtual CPU structure.
3056	*/
3057	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3058	{
3059	/*
3060	* !!! IMPORTANT !!!
3061	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3062	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3063	*/
3064	VMMR0AssertionRemoveNotification(pVCpu);
3065	VMMRZCallRing3Disable(pVCpu);
3066	HM_DISABLE_PREEMPT(pVCpu);
3067
3068	/* Import the entire guest state. */
3069	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3070
3071	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3072	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3073
3074	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3075	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3076
3077	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3078	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3079	VMMR0ThreadCtxHookDisable(pVCpu);
3080
3081	/* Leave HM context. This takes care of local init (term). */
3082	HMR0LeaveCpu(pVCpu);
3083
3084	HM_RESTORE_PREEMPT();
3085	return VINF_SUCCESS;
3086	}
3087
3088
3089	/**
3090	* Take necessary actions before going back to ring-3.
3091	*
3092	* An action requires us to go back to ring-3. This function does the necessary
3093	* steps before we can safely return to ring-3. This is not the same as longjmps
3094	* to ring-3, this is voluntary.
3095	*
3096	* @returns Strict VBox status code.
3097	* @param pVCpu The cross context virtual CPU structure.
3098	* @param rcExit The reason for exiting to ring-3. Can be
3099	* VINF_VMM_UNKNOWN_RING3_CALL.
3100	*/
3101	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3102	{
3103	Assert(pVCpu);
3104	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3105
3106	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3107	VMMRZCallRing3Disable(pVCpu);
3108	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3109	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3110
3111	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3112	if (pVCpu->hm.s.Event.fPending)
3113	{
3114	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3115	Assert(!pVCpu->hm.s.Event.fPending);
3116	}
3117
3118	/* Sync. the necessary state for going back to ring-3. */
3119	hmR0SvmLeaveSession(pVCpu);
3120	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3121
3122	/* Thread-context hooks are unregistered at this point!!! */
3123	/* Ring-3 callback notifications are unregistered at this point!!! */
3124
3125	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3126	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3127	\| CPUM_CHANGED_LDTR
3128	\| CPUM_CHANGED_GDTR
3129	\| CPUM_CHANGED_IDTR
3130	\| CPUM_CHANGED_TR
3131	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3132	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3133	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3134	{
3135	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3136	}
3137
3138	/* Update the exit-to-ring 3 reason. */
3139	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3140
3141	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3142	if ( rcExit != VINF_EM_RAW_INTERRUPT
3143	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3144	{
3145	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3146	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3147	}
3148
3149	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3150	VMMRZCallRing3Enable(pVCpu);
3151
3152	/*
3153	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3154	* and if we're injecting an event we should have a TRPM trap pending.
3155	*/
3156	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3157	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3158	VERR_SVM_IPE_5);
3159	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3160	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3161	VERR_SVM_IPE_4);
3162
3163	return rcExit;
3164	}
3165
3166
3167	/**
3168	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3169	* intercepts.
3170	*
3171	* @param pVCpu The cross context virtual CPU structure.
3172	* @param pVmcb Pointer to the VM control block.
3173	*
3174	* @remarks No-long-jump zone!!!
3175	*/
3176	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3177	{
3178	/*
3179	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3180	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3181	* RDTSC/RDTSCP as well.
3182	*/
3183	bool fParavirtTsc;
3184	uint64_t uTscOffset;
3185	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3186
3187	bool fIntercept;
3188	if (fCanUseRealTsc)
3189	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3190	else
3191	{
3192	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3193	fIntercept = true;
3194	}
3195
3196	if (!fIntercept)
3197	{
3198	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3199	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3200	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3201	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3202	#endif
3203
3204	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3205	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3206	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3207	}
3208
3209	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3210	information before every VM-entry, hence we have nothing to do here at the moment. */
3211	if (fParavirtTsc)
3212	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3213	}
3214
3215
3216	/**
3217	* Sets an event as a pending event to be injected into the guest.
3218	*
3219	* @param pVCpu The cross context virtual CPU structure.
3220	* @param pEvent Pointer to the SVM event.
3221	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3222	* page-fault.
3223	*
3224	* @remarks Statistics counter assumes this is a guest event being reflected to
3225	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3226	*/
3227	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3228	{
3229	Assert(!pVCpu->hm.s.Event.fPending);
3230	Assert(pEvent->n.u1Valid);
3231
3232	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3233	pVCpu->hm.s.Event.fPending = true;
3234	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3235
3236	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3237	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3238	}
3239
3240
3241	/**
3242	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3243	*
3244	* @param pVCpu The cross context virtual CPU structure.
3245	*/
3246	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3247	{
3248	SVMEVENT Event;
3249	Event.u = 0;
3250	Event.n.u1Valid = 1;
3251	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3252	Event.n.u8Vector = X86_XCPT_UD;
3253	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3254	}
3255
3256
3257	/**
3258	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3259	*
3260	* @param pVCpu The cross context virtual CPU structure.
3261	*/
3262	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3263	{
3264	SVMEVENT Event;
3265	Event.u = 0;
3266	Event.n.u1Valid = 1;
3267	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3268	Event.n.u8Vector = X86_XCPT_DB;
3269	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3270	}
3271
3272
3273	/**
3274	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3275	*
3276	* @param pVCpu The cross context virtual CPU structure.
3277	* @param u32ErrCode The error-code for the page-fault.
3278	* @param uFaultAddress The page fault address (CR2).
3279	*
3280	* @remarks This updates the guest CR2 with @a uFaultAddress!
3281	*/
3282	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3283	{
3284	SVMEVENT Event;
3285	Event.u = 0;
3286	Event.n.u1Valid = 1;
3287	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3288	Event.n.u8Vector = X86_XCPT_PF;
3289	Event.n.u1ErrorCodeValid = 1;
3290	Event.n.u32ErrorCode = u32ErrCode;
3291
3292	/* Update CR2 of the guest. */
3293	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3294	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3295	{
3296	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3297	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3298	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3299	}
3300
3301	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3302	}
3303
3304
3305	/**
3306	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3307	*
3308	* @param pVCpu The cross context virtual CPU structure.
3309	*/
3310	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3311	{
3312	SVMEVENT Event;
3313	Event.u = 0;
3314	Event.n.u1Valid = 1;
3315	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3316	Event.n.u8Vector = X86_XCPT_MF;
3317	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3318	}
3319
3320
3321	/**
3322	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3323	*
3324	* @param pVCpu The cross context virtual CPU structure.
3325	*/
3326	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3327	{
3328	SVMEVENT Event;
3329	Event.u = 0;
3330	Event.n.u1Valid = 1;
3331	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3332	Event.n.u8Vector = X86_XCPT_DF;
3333	Event.n.u1ErrorCodeValid = 1;
3334	Event.n.u32ErrorCode = 0;
3335	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3336	}
3337
3338
3339	/**
3340	* Injects an event into the guest upon VMRUN by updating the relevant field
3341	* in the VMCB.
3342	*
3343	* @param pVCpu The cross context virtual CPU structure.
3344	* @param pVmcb Pointer to the guest VM control block.
3345	* @param pEvent Pointer to the event.
3346	*
3347	* @remarks No-long-jump zone!!!
3348	* @remarks Requires CR0!
3349	*/
3350	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3351	{
3352	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3353	pVmcb->ctrl.EventInject.u = pEvent->u;
3354	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3355	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3356	{
3357	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3358	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3359	}
3360	else
3361	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3362	RT_NOREF(pVCpu);
3363
3364	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3365	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3366	}
3367
3368
3369
3370	/**
3371	* Converts any TRPM trap into a pending HM event. This is typically used when
3372	* entering from ring-3 (not longjmp returns).
3373	*
3374	* @param pVCpu The cross context virtual CPU structure.
3375	*/
3376	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3377	{
3378	Assert(TRPMHasTrap(pVCpu));
3379	Assert(!pVCpu->hm.s.Event.fPending);
3380
3381	uint8_t uVector;
3382	TRPMEVENT enmTrpmEvent;
3383	uint32_t uErrCode;
3384	RTGCUINTPTR GCPtrFaultAddress;
3385	uint8_t cbInstr;
3386
3387	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3388	AssertRC(rc);
3389
3390	SVMEVENT Event;
3391	Event.u = 0;
3392	Event.n.u1Valid = 1;
3393	Event.n.u8Vector = uVector;
3394
3395	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3396	if (enmTrpmEvent == TRPM_TRAP)
3397	{
3398	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3399	switch (uVector)
3400	{
3401	case X86_XCPT_NMI:
3402	{
3403	Event.n.u3Type = SVM_EVENT_NMI;
3404	break;
3405	}
3406
3407	case X86_XCPT_BP:
3408	case X86_XCPT_OF:
3409	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3410	RT_FALL_THRU();
3411
3412	case X86_XCPT_PF:
3413	case X86_XCPT_DF:
3414	case X86_XCPT_TS:
3415	case X86_XCPT_NP:
3416	case X86_XCPT_SS:
3417	case X86_XCPT_GP:
3418	case X86_XCPT_AC:
3419	{
3420	Event.n.u1ErrorCodeValid = 1;
3421	Event.n.u32ErrorCode = uErrCode;
3422	break;
3423	}
3424	}
3425	}
3426	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3427	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3428	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3429	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3430	else
3431	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3432
3433	rc = TRPMResetTrap(pVCpu);
3434	AssertRC(rc);
3435
3436	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3437	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3438
3439	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3440	}
3441
3442
3443	/**
3444	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3445	* AMD-V to execute any instruction.
3446	*
3447	* @param pVCpu The cross context virtual CPU structure.
3448	*/
3449	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3450	{
3451	Assert(pVCpu->hm.s.Event.fPending);
3452	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3453
3454	SVMEVENT Event;
3455	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3456
3457	uint8_t uVector = Event.n.u8Vector;
3458	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3459
3460	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3461
3462	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3463	AssertRC(rc);
3464
3465	if (Event.n.u1ErrorCodeValid)
3466	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3467
3468	if ( enmTrapType == TRPM_TRAP
3469	&& uVector == X86_XCPT_PF)
3470	{
3471	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3472	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3473	}
3474	else if (enmTrapType == TRPM_SOFTWARE_INT)
3475	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3476	pVCpu->hm.s.Event.fPending = false;
3477	}
3478
3479
3480	/**
3481	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3482	* now.
3483	*
3484	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3485	* @param pVCpu The cross context virtual CPU structure.
3486	*
3487	* @remarks No-long-jump zone!!!
3488	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3489	*/
3490	static bool hmR0SvmIsIntrShadowActive(PVMCPUCC pVCpu)
3491	{
3492	/*
3493	* Instructions like STI and MOV SS inhibit interrupts till the next instruction
3494	* completes. Check if we should inhibit interrupts or clear any existing
3495	* interrupt inhibition.
3496	*/
3497	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3498	{
3499	if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
3500	{
3501	/*
3502	* We can clear the inhibit force flag as even if we go back to the recompiler
3503	* without executing guest code in AMD-V, the flag's condition to be cleared is
3504	* met and thus the cleared state is correct.
3505	*/
3506	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3507	return false;
3508	}
3509	return true;
3510	}
3511	return false;
3512	}
3513
3514
3515	/**
3516	* Sets the virtual interrupt intercept control in the VMCB.
3517	*
3518	* @param pVCpu The cross context virtual CPU structure.
3519	* @param pVmcb Pointer to the VM control block.
3520	*/
3521	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3522	{
3523	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3524
3525	/*
3526	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3527	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3528	* (updating ISR at the right time) and inject the interrupt.
3529	*
3530	* With AVIC is supported, we could make use of the asynchronously delivery without
3531	* #VMEXIT and we would be passing the AVIC page to SVM.
3532	*
3533	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3534	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3535	*/
3536	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3537	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3538	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3539	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3540	Log4(("Set VINTR intercept\n"));
3541	}
3542
3543
3544	/**
3545	* Clears the virtual interrupt intercept control in the VMCB as
3546	* we are figured the guest is unable process any interrupts
3547	* at this point of time.
3548	*
3549	* @param pVCpu The cross context virtual CPU structure.
3550	* @param pVmcb Pointer to the VM control block.
3551	*/
3552	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3553	{
3554	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3555
3556	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3557	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3558	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3559	{
3560	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3561	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3562	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3563	Log4(("Cleared VINTR intercept\n"));
3564	}
3565	}
3566
3567
3568	/**
3569	* Evaluates the event to be delivered to the guest and sets it as the pending
3570	* event.
3571	*
3572	* @returns Strict VBox status code.
3573	* @param pVCpu The cross context virtual CPU structure.
3574	* @param pSvmTransient Pointer to the SVM transient structure.
3575	*/
3576	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3577	{
3578	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3579	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3580	\| CPUMCTX_EXTRN_RFLAGS
3581	\| CPUMCTX_EXTRN_INHIBIT_INT
3582	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3583
3584	Assert(!pVCpu->hm.s.Event.fPending);
3585	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3586	Assert(pVmcb);
3587
3588	bool const fGif = CPUMGetGuestGif(pCtx);
3589	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3590	bool const fBlockNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3591
3592	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3593	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3594	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3595
3596	/** @todo SMI. SMIs take priority over NMIs. */
3597
3598	/*
3599	* Check if the guest or nested-guest can receive NMIs.
3600	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3601	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3602	*/
3603	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3604	&& !fBlockNmi)
3605	{
3606	if ( fGif
3607	&& !fIntShadow)
3608	{
3609	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3610	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3611	{
3612	Log4(("Intercepting NMI -> #VMEXIT\n"));
3613	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3614	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3615	}
3616	#endif
3617	Log4(("Setting NMI pending for injection\n"));
3618	SVMEVENT Event;
3619	Event.u = 0;
3620	Event.n.u1Valid = 1;
3621	Event.n.u8Vector = X86_XCPT_NMI;
3622	Event.n.u3Type = SVM_EVENT_NMI;
3623	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3624	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3625	}
3626	else if (!fGif)
3627	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3628	else if (!pSvmTransient->fIsNestedGuest)
3629	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3630	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3631	}
3632	/*
3633	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3634	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3635	* it from the APIC device.
3636	*
3637	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3638	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3639	* do that work when we execute nested-guest code esp. since all the required information
3640	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3641	* the virtual interrupt controller.
3642	*
3643	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3644	*/
3645	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3646	&& !pVCpu->hm.s.fSingleInstruction)
3647	{
3648	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u32 & X86_EFL_IF)
3649	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3650	if ( fGif
3651	&& !fBlockInt
3652	&& !fIntShadow)
3653	{
3654	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3655	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3656	{
3657	Log4(("Intercepting INTR -> #VMEXIT\n"));
3658	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3659	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3660	}
3661	#endif
3662	uint8_t u8Interrupt;
3663	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3664	if (RT_SUCCESS(rc))
3665	{
3666	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3667	SVMEVENT Event;
3668	Event.u = 0;
3669	Event.n.u1Valid = 1;
3670	Event.n.u8Vector = u8Interrupt;
3671	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3672	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3673	}
3674	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3675	{
3676	/*
3677	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3678	* updated eventually when the TPR is written by the guest.
3679	*/
3680	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3681	}
3682	else
3683	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3684	}
3685	else if (!fGif)
3686	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3687	else if (!pSvmTransient->fIsNestedGuest)
3688	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3689	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3690	}
3691
3692	return VINF_SUCCESS;
3693	}
3694
3695
3696	/**
3697	* Injects any pending events into the guest (or nested-guest).
3698	*
3699	* @param pVCpu The cross context virtual CPU structure.
3700	* @param pVmcb Pointer to the VM control block.
3701	*
3702	* @remarks Must only be called when we are guaranteed to enter
3703	* hardware-assisted SVM execution and not return to ring-3
3704	* prematurely.
3705	*/
3706	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3707	{
3708	Assert(!TRPMHasTrap(pVCpu));
3709	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3710
3711	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3712	#ifdef VBOX_STRICT
3713	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3714	bool const fGif = CPUMGetGuestGif(pCtx);
3715	bool fAllowInt = fGif;
3716	if (fGif)
3717	{
3718	/*
3719	* For nested-guests we have no way to determine if we're injecting a physical or
3720	* virtual interrupt at this point. Hence the partial verification below.
3721	*/
3722	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3723	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3724	else
3725	fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
3726	}
3727	#endif
3728
3729	if (pVCpu->hm.s.Event.fPending)
3730	{
3731	SVMEVENT Event;
3732	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3733	Assert(Event.n.u1Valid);
3734
3735	/*
3736	* Validate event injection pre-conditions.
3737	*/
3738	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3739	{
3740	Assert(fAllowInt);
3741	Assert(!fIntShadow);
3742	}
3743	else if (Event.n.u3Type == SVM_EVENT_NMI)
3744	{
3745	Assert(fGif);
3746	Assert(!fIntShadow);
3747	}
3748
3749	/*
3750	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3751	* do this only when we are surely going to inject the NMI as otherwise if we return
3752	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3753	* SVM R0.
3754	*
3755	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3756	* which will set the VMCS field after actually delivering the NMI which we read on
3757	* VM-exit to determine the state.
3758	*/
3759	if ( Event.n.u3Type == SVM_EVENT_NMI
3760	&& Event.n.u8Vector == X86_XCPT_NMI
3761	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3762	{
3763	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3764	}
3765
3766	/*
3767	* Inject it (update VMCB for injection by the hardware).
3768	*/
3769	Log4(("Injecting pending HM event\n"));
3770	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3771	pVCpu->hm.s.Event.fPending = false;
3772
3773	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3774	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3775	else
3776	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3777	}
3778	else
3779	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3780
3781	/*
3782	* We could have injected an NMI through IEM and continue guest execution using
3783	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3784	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3785	*/
3786	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3787	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3788
3789	/*
3790	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3791	*
3792	* For nested-guests: We need to update it too for the scenario where IEM executes
3793	* the nested-guest but execution later continues here with an interrupt shadow active.
3794	*/
3795	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3796	}
3797
3798
3799	/**
3800	* Reports world-switch error and dumps some useful debug info.
3801	*
3802	* @param pVCpu The cross context virtual CPU structure.
3803	* @param rcVMRun The return code from VMRUN (or
3804	* VERR_SVM_INVALID_GUEST_STATE for invalid
3805	* guest-state).
3806	*/
3807	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3808	{
3809	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3810	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3811	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3812
3813	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3814	{
3815	#ifdef VBOX_STRICT
3816	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3817	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3818	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3819	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3820	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3821	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3822	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3823	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3824	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3825	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3826	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3827	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3828
3829	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3830	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3831	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3832
3833	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3834	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3835	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3836	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3837	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3838	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3839	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3840	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3841	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3842	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3843	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3844	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3845
3846	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3847	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3848	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3849	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3850	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3851	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3852	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3853	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3854	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3855	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3856	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3857	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3858	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3859	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3860	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3861	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3862	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3863	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3864	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3865	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3866
3867	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3868
3869	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3870	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3871
3872	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3873	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3874	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3875	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3876	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3877	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3878	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3879	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3880	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3881	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3882	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3883	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3884	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3885	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3886	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3887	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3888	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3889	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3890	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3891	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3892
3893	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3894	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3895
3896	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3897	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3898	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3899	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3900
3901	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3902	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3903
3904	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3905	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3906	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3907	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3908
3909	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3910	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3911	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3912	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3913	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3914	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3915	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3916
3917	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3918	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3919	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3920	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3921
3922	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3923	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3924	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3925
3926	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3927	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3928	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3929	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3930	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3931	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3932	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3933	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3934	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3935	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3936	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3937	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3938
3939	NOREF(pVmcb);
3940	#endif /* VBOX_STRICT */
3941	}
3942	else
3943	Log4Func(("rcVMRun=%d\n", rcVMRun));
3944	}
3945
3946
3947	/**
3948	* Check per-VM and per-VCPU force flag actions that require us to go back to
3949	* ring-3 for one reason or another.
3950	*
3951	* @returns Strict VBox status code (information status code included).
3952	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3953	* ring-3.
3954	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3955	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3956	* interrupts)
3957	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3958	* all EMTs to be in ring-3.
3959	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3960	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3961	* to the EM loop.
3962	*
3963	* @param pVCpu The cross context virtual CPU structure.
3964	*/
3965	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3966	{
3967	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3968
3969	/* Could happen as a result of longjump. */
3970	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3971	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3972
3973	/* Update pending interrupts into the APIC's IRR. */
3974	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3975	APICUpdatePendingInterrupts(pVCpu);
3976
3977	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3978	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3979	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3980	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3981	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3982	{
3983	/* Pending PGM C3 sync. */
3984	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3985	{
3986	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3987	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3988	if (rc != VINF_SUCCESS)
3989	{
3990	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3991	return rc;
3992	}
3993	}
3994
3995	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3996	/* -XXX- what was that about single stepping? */
3997	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3998	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3999	{
4000	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4001	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
4002	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
4003	return rc;
4004	}
4005
4006	/* Pending VM request packets, such as hardware interrupts. */
4007	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4008	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4009	{
4010	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
4011	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4012	return VINF_EM_PENDING_REQUEST;
4013	}
4014
4015	/* Pending PGM pool flushes. */
4016	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4017	{
4018	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4019	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4020	return VINF_PGM_POOL_FLUSH_PENDING;
4021	}
4022
4023	/* Pending DMA requests. */
4024	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4025	{
4026	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4027	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4028	return VINF_EM_RAW_TO_R3;
4029	}
4030	}
4031
4032	return VINF_SUCCESS;
4033	}
4034
4035
4036	/**
4037	* Does the preparations before executing guest code in AMD-V.
4038	*
4039	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4040	* recompiler. We must be cautious what we do here regarding committing
4041	* guest-state information into the VMCB assuming we assuredly execute the guest
4042	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4043	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4044	* that the recompiler can (and should) use them when it resumes guest
4045	* execution. Otherwise such operations must be done when we can no longer
4046	* exit to ring-3.
4047	*
4048	* @returns Strict VBox status code (informational status codes included).
4049	* @retval VINF_SUCCESS if we can proceed with running the guest.
4050	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4051	*
4052	* @param pVCpu The cross context virtual CPU structure.
4053	* @param pSvmTransient Pointer to the SVM transient structure.
4054	*/
4055	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4056	{
4057	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4058
4059	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4060	if (pSvmTransient->fIsNestedGuest)
4061	{
4062	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4063	return VINF_EM_RESCHEDULE_REM;
4064	}
4065	#endif
4066
4067	/* Check force flag actions that might require us to go back to ring-3. */
4068	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4069	if (rc != VINF_SUCCESS)
4070	return rc;
4071
4072	if (TRPMHasTrap(pVCpu))
4073	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4074	else if (!pVCpu->hm.s.Event.fPending)
4075	{
4076	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4077	if ( rc != VINF_SUCCESS
4078	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4079	{
4080	/* If a nested-guest VM-exit occurred, bail. */
4081	if (pSvmTransient->fIsNestedGuest)
4082	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4083	return rc;
4084	}
4085	}
4086
4087	/*
4088	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4089	* Just do it in software, see @bugref{8411}.
4090	* NB: If we could continue a task switch exit we wouldn't need to do this.
4091	*/
4092	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4093	if (RT_UNLIKELY( !g_fHmSvmFeatures
4094	&& pVCpu->hm.s.Event.fPending
4095	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4096	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4097
4098	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4099	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4100	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4101	#endif
4102
4103	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4104	/*
4105	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4106	*/
4107	if (pSvmTransient->fIsNestedGuest)
4108	hmR0SvmSetupVmcbNested(pVCpu);
4109	#endif
4110
4111	/*
4112	* Export the guest state bits that are not shared with the host in any way as we can
4113	* longjmp or get preempted in the midst of exporting some of the state.
4114	*/
4115	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4116	AssertRCReturn(rc, rc);
4117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4118
4119	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4120	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4121
4122	/*
4123	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4124	* world-switch so we can update it on the way back if the guest changed the TPR.
4125	*/
4126	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4127	{
4128	Assert(!pSvmTransient->fIsNestedGuest);
4129	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4130	if (pVM->hm.s.fTprPatchingActive)
4131	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4132	else
4133	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4134	}
4135
4136	/*
4137	* No longjmps to ring-3 from this point on!!!
4138	*
4139	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4140	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4141	*/
4142	VMMRZCallRing3Disable(pVCpu);
4143
4144	/*
4145	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4146	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4147	* preemption disabled for a while. Since this is purly to aid the
4148	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4149	* disable interrupt on NT.
4150	*
4151	* We need to check for force-flags that could've possible been altered since we last
4152	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4153	* see @bugref{6398}).
4154	*
4155	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4156	* to ring-3 before executing guest code.
4157	*/
4158	pSvmTransient->fEFlags = ASMIntDisableFlags();
4159	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4160	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4161	{
4162	ASMSetFlags(pSvmTransient->fEFlags);
4163	VMMRZCallRing3Enable(pVCpu);
4164	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4165	return VINF_EM_RAW_TO_R3;
4166	}
4167	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4168	{
4169	ASMSetFlags(pSvmTransient->fEFlags);
4170	VMMRZCallRing3Enable(pVCpu);
4171	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4172	return VINF_EM_RAW_INTERRUPT;
4173	}
4174
4175	return VINF_SUCCESS;
4176	}
4177
4178
4179	/**
4180	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4181	* doing so.
4182	*
4183	* This means there is no backing out to ring-3 or anywhere else at this point.
4184	*
4185	* @param pVCpu The cross context virtual CPU structure.
4186	* @param pSvmTransient Pointer to the SVM transient structure.
4187	*
4188	* @remarks Called with preemption disabled.
4189	* @remarks No-long-jump zone!!!
4190	*/
4191	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4192	{
4193	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4194	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4195
4196	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4197	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4198
4199	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4200	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4201
4202	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4203
4204	if (!CPUMIsGuestFPUStateActive(pVCpu))
4205	{
4206	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4207	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4208	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4209	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4210	}
4211
4212	/* Load the state shared between host and guest (FPU, debug). */
4213	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4214	hmR0SvmExportSharedState(pVCpu, pVmcb);
4215
4216	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4217	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4218
4219	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4220	RTCPUID const idHostCpu = pHostCpu->idCpu;
4221	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4222
4223	/* Setup TSC offsetting. */
4224	if ( pSvmTransient->fUpdateTscOffsetting
4225	\|\| fMigratedHostCpu)
4226	{
4227	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4228	pSvmTransient->fUpdateTscOffsetting = false;
4229	}
4230
4231	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4232	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4233	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4234	else
4235	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4236
4237	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4238	if (fMigratedHostCpu)
4239	pVmcb->ctrl.u32VmcbCleanBits = 0;
4240
4241	/* Store status of the shared guest-host state at the time of VMRUN. */
4242	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4243	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4244
4245	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4246	uint8_t *pbMsrBitmap;
4247	if (!pSvmTransient->fIsNestedGuest)
4248	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4249	else
4250	{
4251	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4252	* MSRPM and only perform this if it changed also use EVEX.POR when it
4253	* does. */
4254	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4255
4256	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4257	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4258	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4259	}
4260	#else
4261	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4262	#endif
4263
4264	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4265	/* Flush the appropriate tagged-TLB entries. */
4266	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4267	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4268
4269	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4270
4271	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4272	to start executing. */
4273
4274	/*
4275	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4276	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4277	*
4278	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4279	*/
4280	if ( pVM->cpum.ro.HostFeatures.fRdTscP
4281	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4282	{
4283	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4284	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4285	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4286	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4287	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4288	pSvmTransient->fRestoreTscAuxMsr = true;
4289	}
4290	else
4291	{
4292	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4293	pSvmTransient->fRestoreTscAuxMsr = false;
4294	}
4295	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4296
4297	/*
4298	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4299	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4300	* from the VMCB.
4301	*/
4302	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4303	if (!fSupportsVmcbCleanBits)
4304	pVmcb->ctrl.u32VmcbCleanBits = 0;
4305	}
4306
4307
4308	/**
4309	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4310	*
4311	* @returns VBox strict status code.
4312	* @param pVCpu The cross context virtual CPU structure.
4313	* @param HCPhysVmcb The host physical address of the VMCB.
4314	*
4315	* @remarks No-long-jump zone!!!
4316	*/
4317	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4318	{
4319	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4320	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4321	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4322	}
4323
4324
4325	/**
4326	* Performs some essential restoration of state after running guest (or
4327	* nested-guest) code in AMD-V.
4328	*
4329	* @param pVCpu The cross context virtual CPU structure.
4330	* @param pSvmTransient Pointer to the SVM transient structure.
4331	* @param rcVMRun Return code of VMRUN.
4332	*
4333	* @remarks Called with interrupts disabled.
4334	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4335	* unconditionally when it is safe to do so.
4336	*/
4337	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4338	{
4339	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4340
4341	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4342	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4343
4344	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4345	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4346
4347	/* TSC read must be done early for maximum accuracy. */
4348	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4349	{
4350	if (!pSvmTransient->fIsNestedGuest)
4351	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4352	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4353	else
4354	{
4355	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4356	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4357	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4358	}
4359	#endif
4360	}
4361
4362	if (pSvmTransient->fRestoreTscAuxMsr)
4363	{
4364	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4365	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4366	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4367	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4368	}
4369
4370	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4371	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4372	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4373	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4374
4375	Assert(!(ASMGetFlags() & X86_EFL_IF));
4376	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4377	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4378
4379	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4380	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4381	{
4382	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4383	return;
4384	}
4385
4386	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4387	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4388	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4389	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4390
4391	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4392	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4393	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4394	#else
4395	/*
4396	* Always import the following:
4397	*
4398	* - RIP for exit optimizations and evaluating event injection on re-entry.
4399	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4400	* state on preemption.
4401	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4402	* - CS for exit optimizations.
4403	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4404	* assembly switcher code.
4405	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4406	*/
4407	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4408	\| CPUMCTX_EXTRN_RFLAGS
4409	\| CPUMCTX_EXTRN_RAX
4410	\| CPUMCTX_EXTRN_RSP
4411	\| CPUMCTX_EXTRN_CS
4412	\| CPUMCTX_EXTRN_HWVIRT
4413	\| CPUMCTX_EXTRN_INHIBIT_INT
4414	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4415	\| HMSVM_CPUMCTX_SHARED_STATE);
4416	#endif
4417
4418	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4419	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4420	{
4421	Assert(!pSvmTransient->fIsNestedGuest);
4422	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4423	if ( pVM->hm.s.fTprPatchingActive
4424	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4425	{
4426	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4427	AssertRC(rc);
4428	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4429	}
4430	/* Sync TPR when we aren't intercepting CR8 writes. */
4431	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4432	{
4433	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4434	AssertRC(rc);
4435	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4436	}
4437	}
4438
4439	#ifdef DEBUG_ramshankar
4440	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4441	{
4442	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4443	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4444	0 /* uVerbose */);
4445	}
4446	#endif
4447
4448	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4449	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4450	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4451	}
4452
4453
4454	/**
4455	* Runs the guest code using AMD-V.
4456	*
4457	* @returns Strict VBox status code.
4458	* @param pVCpu The cross context virtual CPU structure.
4459	* @param pcLoops Pointer to the number of executed loops.
4460	*/
4461	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4462	{
4463	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4464	Assert(pcLoops);
4465	Assert(*pcLoops <= cMaxResumeLoops);
4466
4467	SVMTRANSIENT SvmTransient;
4468	RT_ZERO(SvmTransient);
4469	SvmTransient.fUpdateTscOffsetting = true;
4470	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4471
4472	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4473	for (;;)
4474	{
4475	Assert(!HMR0SuspendPending());
4476	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4477
4478	/* Preparatory work for running nested-guest code, this may force us to return to
4479	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4480	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4481	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4482	if (rc != VINF_SUCCESS)
4483	break;
4484
4485	/*
4486	* No longjmps to ring-3 from this point on!!!
4487	*
4488	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4489	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4490	*/
4491	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4492	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4493
4494	/* Restore any residual host-state and save any bits shared between host and guest
4495	into the guest-CPU state. Re-enables interrupts! */
4496	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4497
4498	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4499	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4500	{
4501	if (rc == VINF_SUCCESS)
4502	rc = VERR_SVM_INVALID_GUEST_STATE;
4503	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4504	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4505	break;
4506	}
4507
4508	/* Handle the #VMEXIT. */
4509	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4510	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4511	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4512	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4513	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4514	if (rc != VINF_SUCCESS)
4515	break;
4516	if (++(*pcLoops) >= cMaxResumeLoops)
4517	{
4518	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4519	rc = VINF_EM_RAW_INTERRUPT;
4520	break;
4521	}
4522	}
4523
4524	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4525	return rc;
4526	}
4527
4528
4529	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4530	/**
4531	* Runs the nested-guest code using AMD-V.
4532	*
4533	* @returns Strict VBox status code.
4534	* @param pVCpu The cross context virtual CPU structure.
4535	* @param pcLoops Pointer to the number of executed loops. If we're switching
4536	* from the guest-code execution loop to this nested-guest
4537	* execution loop pass the remainder value, else pass 0.
4538	*/
4539	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4540	{
4541	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4542	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4543	Assert(pcLoops);
4544	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4545	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4546	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4547
4548	SVMTRANSIENT SvmTransient;
4549	RT_ZERO(SvmTransient);
4550	SvmTransient.fUpdateTscOffsetting = true;
4551	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4552	SvmTransient.fIsNestedGuest = true;
4553
4554	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4555	for (;;)
4556	{
4557	Assert(!HMR0SuspendPending());
4558	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4559
4560	/* Preparatory work for running nested-guest code, this may force us to return to
4561	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4562	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4563	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4564	if ( rc != VINF_SUCCESS
4565	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4566	break;
4567
4568	/*
4569	* No longjmps to ring-3 from this point on!!!
4570	*
4571	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4572	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4573	*/
4574	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4575
4576	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4577
4578	/* Restore any residual host-state and save any bits shared between host and guest
4579	into the guest-CPU state. Re-enables interrupts! */
4580	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4581
4582	if (RT_LIKELY( rc == VINF_SUCCESS
4583	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4584	{ /* extremely likely */ }
4585	else
4586	{
4587	/* VMRUN failed, shouldn't really happen, Guru. */
4588	if (rc != VINF_SUCCESS)
4589	break;
4590
4591	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4592	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4593	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4594	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4595	break;
4596	}
4597
4598	/* Handle the #VMEXIT. */
4599	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4600	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4601	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4602	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4603	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4604	if (rc == VINF_SUCCESS)
4605	{
4606	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4607	{
4608	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4609	rc = VINF_SVM_VMEXIT;
4610	}
4611	else
4612	{
4613	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4614	continue;
4615	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4616	rc = VINF_EM_RAW_INTERRUPT;
4617	}
4618	}
4619	else
4620	Assert(rc != VINF_SVM_VMEXIT);
4621	break;
4622	/** @todo NSTSVM: handle single-stepping. */
4623	}
4624
4625	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4626	return rc;
4627	}
4628	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4629
4630
4631	/**
4632	* Checks if any expensive dtrace probes are enabled and we should go to the
4633	* debug loop.
4634	*
4635	* @returns true if we should use debug loop, false if not.
4636	*/
4637	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4638	{
4639	/* It's probably faster to OR the raw 32-bit counter variables together.
4640	Since the variables are in an array and the probes are next to one
4641	another (more or less), we have good locality. So, better read
4642	eight-nine cache lines ever time and only have one conditional, than
4643	128+ conditionals, right? */
4644	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4645	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4648	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4649	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4650	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4651	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4652	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4653	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4654	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4655	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4656	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4657	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4658	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4659	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4660	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4661	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4662	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4663	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4664	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4665	) != 0
4666	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4677	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4679	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4680	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4681	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4682	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4683	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4684	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4685	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4686	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4687	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4689	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4690	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4691	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4692	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4693	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4694	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4695	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4696	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4697	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4698	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4699	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4700	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4701	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4702	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4703	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4704	) != 0
4705	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4717	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4719	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4720	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4721	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4722	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4723	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4724	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4725	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4726	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4729	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4730	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4731	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4732	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4733	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4734	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4735	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4736	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4737	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4738	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4739	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4740	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4741	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4742	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4743	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4744	) != 0;
4745	}
4746
4747
4748	/**
4749	* Runs the guest code using AMD-V.
4750	*
4751	* @returns Strict VBox status code.
4752	* @param pVCpu The cross context virtual CPU structure.
4753	*/
4754	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4755	{
4756	AssertPtr(pVCpu);
4757	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4758	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4759	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4760	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4761
4762	uint32_t cLoops = 0;
4763	VBOXSTRICTRC rc;
4764	for (;;)
4765	{
4766	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4767	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4768	#else
4769	NOREF(pCtx);
4770	bool const fInNestedGuestMode = false;
4771	#endif
4772	if (!fInNestedGuestMode)
4773	{
4774	if ( !pVCpu->hm.s.fUseDebugLoop
4775	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4776	&& !DBGFIsStepping(pVCpu)
4777	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4778	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4779	else
4780	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4781	}
4782	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4783	else
4784	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4785
4786	if (rc == VINF_SVM_VMRUN)
4787	{
4788	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4789	continue;
4790	}
4791	if (rc == VINF_SVM_VMEXIT)
4792	{
4793	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4794	continue;
4795	}
4796	#endif
4797	break;
4798	}
4799
4800	/* Fixup error codes. */
4801	if (rc == VERR_EM_INTERPRETER)
4802	rc = VINF_EM_RAW_EMULATE_INSTR;
4803	else if (rc == VINF_EM_RESET)
4804	rc = VINF_EM_TRIPLE_FAULT;
4805
4806	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4807	rc = hmR0SvmExitToRing3(pVCpu, rc);
4808	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4809	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4810	return rc;
4811	}
4812
4813	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4814
4815	/**
4816	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4817	*
4818	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4819	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4820	*/
4821	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4822	{
4823	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4824	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4825	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4826	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4827	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4828	const bool fRep = pIoExitInfo->n.u1Rep;
4829	const bool fStrIo = pIoExitInfo->n.u1Str;
4830
4831	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4832	NULL /* pIoExitInfo */);
4833	}
4834
4835
4836	/**
4837	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4838	* SVM_EXIT_INVALID).
4839	*
4840	* @returns VBox status code (informational status codes included).
4841	* @param pVCpu The cross context virtual CPU structure.
4842	* @param pSvmTransient Pointer to the SVM transient structure.
4843	*/
4844	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4845	{
4846	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4847	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4848	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4849
4850	/*
4851	* We import the complete state here because we use separate VMCBs for the guest and the
4852	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4853	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4854	*/
4855	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4856	do { \
4857	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4858	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4859	} while (0)
4860
4861	/*
4862	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4863	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4864	*/
4865	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4866	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4867	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4868	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4869	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4870	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4871
4872	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4873	switch (uExitCode)
4874	{
4875	case SVM_EXIT_CPUID:
4876	{
4877	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4878	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4879	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4880	}
4881
4882	case SVM_EXIT_RDTSC:
4883	{
4884	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4885	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4886	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4887	}
4888
4889	case SVM_EXIT_RDTSCP:
4890	{
4891	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4892	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4893	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4894	}
4895
4896	case SVM_EXIT_MONITOR:
4897	{
4898	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4899	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4900	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4901	}
4902
4903	case SVM_EXIT_MWAIT:
4904	{
4905	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4906	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4907	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4908	}
4909
4910	case SVM_EXIT_HLT:
4911	{
4912	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4913	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4914	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4915	}
4916
4917	case SVM_EXIT_MSR:
4918	{
4919	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4920	{
4921	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4922	uint16_t offMsrpm;
4923	uint8_t uMsrpmBit;
4924	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4925	if (RT_SUCCESS(rc))
4926	{
4927	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4928	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4929
4930	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4931	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4932	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4933
4934	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4935	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4936	{
4937	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4938	}
4939	}
4940	else
4941	{
4942	/*
4943	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4944	* See AMD-V spec. "15.11 MSR Intercepts".
4945	*/
4946	Assert(rc == VERR_OUT_OF_RANGE);
4947	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4948	}
4949	}
4950	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4951	}
4952
4953	case SVM_EXIT_IOIO:
4954	{
4955	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4956	{
4957	SVMIOIOEXITINFO IoExitInfo;
4958	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4959	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4960	if (fIntercept)
4961	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4962	}
4963	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4964	}
4965
4966	case SVM_EXIT_XCPT_PF:
4967	{
4968	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4969	if (pVM->hmr0.s.fNestedPaging)
4970	{
4971	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4972	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4973
4974	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4975	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4976	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4977
4978	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4979	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4980	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4981	return VINF_SUCCESS;
4982	}
4983	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4984	}
4985
4986	case SVM_EXIT_XCPT_UD:
4987	{
4988	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4989	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4990	hmR0SvmSetPendingXcptUD(pVCpu);
4991	return VINF_SUCCESS;
4992	}
4993
4994	case SVM_EXIT_XCPT_MF:
4995	{
4996	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4997	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4998	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4999	}
5000
5001	case SVM_EXIT_XCPT_DB:
5002	{
5003	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
5004	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5005	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
5006	}
5007
5008	case SVM_EXIT_XCPT_AC:
5009	{
5010	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5011	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5012	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5013	}
5014
5015	case SVM_EXIT_XCPT_BP:
5016	{
5017	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5018	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5019	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5020	}
5021
5022	case SVM_EXIT_READ_CR0:
5023	case SVM_EXIT_READ_CR3:
5024	case SVM_EXIT_READ_CR4:
5025	{
5026	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5027	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5028	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5029	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5030	}
5031
5032	case SVM_EXIT_CR0_SEL_WRITE:
5033	{
5034	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5035	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5036	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5037	}
5038
5039	case SVM_EXIT_WRITE_CR0:
5040	case SVM_EXIT_WRITE_CR3:
5041	case SVM_EXIT_WRITE_CR4:
5042	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5043	{
5044	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5045	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5046
5047	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5048	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5049	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5050	}
5051
5052	case SVM_EXIT_PAUSE:
5053	{
5054	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5055	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5056	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5057	}
5058
5059	case SVM_EXIT_VINTR:
5060	{
5061	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5062	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5063	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5064	}
5065
5066	case SVM_EXIT_INTR:
5067	case SVM_EXIT_NMI:
5068	case SVM_EXIT_SMI:
5069	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5070	{
5071	/*
5072	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5073	*
5074	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5075	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5076	* nested-guest #VMEXIT.
5077	*
5078	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5079	* while trying to inject interrupts, see comment at the top of this function.
5080	*/
5081	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5082	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5083	}
5084
5085	case SVM_EXIT_FERR_FREEZE:
5086	{
5087	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5088	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5089	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5090	}
5091
5092	case SVM_EXIT_INVLPG:
5093	{
5094	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5095	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5096	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5097	}
5098
5099	case SVM_EXIT_WBINVD:
5100	{
5101	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5102	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5103	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5104	}
5105
5106	case SVM_EXIT_INVD:
5107	{
5108	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5109	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5110	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5111	}
5112
5113	case SVM_EXIT_RDPMC:
5114	{
5115	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5116	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5117	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5118	}
5119
5120	default:
5121	{
5122	switch (uExitCode)
5123	{
5124	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5125	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5126	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5127	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5128	{
5129	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5130	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5131	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5132	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5133	}
5134
5135	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5136	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5137	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5138	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5139	{
5140	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5141	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5142	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5143	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5144	}
5145
5146	case SVM_EXIT_XCPT_DE:
5147	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5148	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5149	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5150	case SVM_EXIT_XCPT_OF:
5151	case SVM_EXIT_XCPT_BR:
5152	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5153	case SVM_EXIT_XCPT_NM:
5154	case SVM_EXIT_XCPT_DF:
5155	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5156	case SVM_EXIT_XCPT_TS:
5157	case SVM_EXIT_XCPT_NP:
5158	case SVM_EXIT_XCPT_SS:
5159	case SVM_EXIT_XCPT_GP:
5160	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5161	case SVM_EXIT_XCPT_15: /* Reserved. */
5162	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5163	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5164	case SVM_EXIT_XCPT_MC:
5165	case SVM_EXIT_XCPT_XF:
5166	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5167	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5168	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5169	{
5170	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5171	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5172	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5173	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5174	}
5175
5176	case SVM_EXIT_XSETBV:
5177	{
5178	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5179	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5180	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5181	}
5182
5183	case SVM_EXIT_TASK_SWITCH:
5184	{
5185	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5186	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5187	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5188	}
5189
5190	case SVM_EXIT_IRET:
5191	{
5192	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5193	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5194	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5195	}
5196
5197	case SVM_EXIT_SHUTDOWN:
5198	{
5199	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5200	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5201	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5202	}
5203
5204	case SVM_EXIT_VMMCALL:
5205	{
5206	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5207	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5208	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5209	}
5210
5211	case SVM_EXIT_CLGI:
5212	{
5213	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5214	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5215	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5216	}
5217
5218	case SVM_EXIT_STGI:
5219	{
5220	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5221	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5222	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5223	}
5224
5225	case SVM_EXIT_VMLOAD:
5226	{
5227	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5228	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5229	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5230	}
5231
5232	case SVM_EXIT_VMSAVE:
5233	{
5234	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5235	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5236	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5237	}
5238
5239	case SVM_EXIT_INVLPGA:
5240	{
5241	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5242	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5243	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5244	}
5245
5246	case SVM_EXIT_VMRUN:
5247	{
5248	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5249	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5250	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5251	}
5252
5253	case SVM_EXIT_RSM:
5254	{
5255	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5256	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5257	hmR0SvmSetPendingXcptUD(pVCpu);
5258	return VINF_SUCCESS;
5259	}
5260
5261	case SVM_EXIT_SKINIT:
5262	{
5263	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5264	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5265	hmR0SvmSetPendingXcptUD(pVCpu);
5266	return VINF_SUCCESS;
5267	}
5268
5269	case SVM_EXIT_NPF:
5270	{
5271	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5272	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5273	}
5274
5275	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5276	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5277
5278	default:
5279	{
5280	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5281	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5282	return VERR_SVM_UNKNOWN_EXIT;
5283	}
5284	}
5285	}
5286	}
5287	/* not reached */
5288
5289	# undef NST_GST_VMEXIT_CALL_RET
5290	}
5291
5292	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5293
5294	/** @def VMEXIT_CALL_RET
5295	* Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5296	*/
5297	#ifdef DEBUG_ramshankar
5298	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5299	do { \
5300	if ((a_fDbg) == 1) \
5301	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5302	int rc = a_CallExpr; \
5303	if ((a_fDbg) == 1) \
5304	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5305	return rc; \
5306	} while (0)
5307	#else
5308	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5309	#endif
5310
5311	/**
5312	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5313	*
5314	* @returns Strict VBox status code (informational status codes included).
5315	* @param pVCpu The cross context virtual CPU structure.
5316	* @param pSvmTransient Pointer to the SVM transient structure.
5317	*/
5318	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5319	{
5320	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5321	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5322
5323	/*
5324	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5325	* for most guests under normal workloads (for some definition of "normal").
5326	*/
5327	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5328	switch (uExitCode)
5329	{
5330	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5331	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5332	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5333	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5334	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5335	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5336	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5337	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5338	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5339	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5340
5341	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5342	case SVM_EXIT_INTR:
5343	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5344
5345	case SVM_EXIT_READ_CR0:
5346	case SVM_EXIT_READ_CR3:
5347	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5348
5349	case SVM_EXIT_CR0_SEL_WRITE:
5350	case SVM_EXIT_WRITE_CR0:
5351	case SVM_EXIT_WRITE_CR3:
5352	case SVM_EXIT_WRITE_CR4:
5353	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5354
5355	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5356	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5357	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5358	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5359	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5360	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5361	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5362	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5363	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5364	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5365	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5366	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5367	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5368	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5369	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5370	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5371
5372	default:
5373	{
5374	switch (pSvmTransient->u64ExitCode)
5375	{
5376	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5377	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5378	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5379	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5380	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5381
5382	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5383	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5384	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5385	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5386	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5387
5388	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5389	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5390
5391	case SVM_EXIT_SMI:
5392	case SVM_EXIT_INIT:
5393	{
5394	/*
5395	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5396	* If it ever does, we want to know about it so log the exit code and bail.
5397	*/
5398	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5399	}
5400
5401	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5402	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5403	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5404	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5405	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5406	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5407	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5408	#else
5409	case SVM_EXIT_CLGI:
5410	case SVM_EXIT_STGI:
5411	case SVM_EXIT_VMLOAD:
5412	case SVM_EXIT_VMSAVE:
5413	case SVM_EXIT_INVLPGA:
5414	case SVM_EXIT_VMRUN:
5415	#endif
5416	case SVM_EXIT_RSM:
5417	case SVM_EXIT_SKINIT:
5418	{
5419	hmR0SvmSetPendingXcptUD(pVCpu);
5420	return VINF_SUCCESS;
5421	}
5422
5423	/*
5424	* The remaining should only be possible when debugging or dtracing.
5425	*/
5426	case SVM_EXIT_XCPT_DE:
5427	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5428	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5429	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5430	case SVM_EXIT_XCPT_OF:
5431	case SVM_EXIT_XCPT_BR:
5432	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5433	case SVM_EXIT_XCPT_NM:
5434	case SVM_EXIT_XCPT_DF:
5435	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5436	case SVM_EXIT_XCPT_TS:
5437	case SVM_EXIT_XCPT_NP:
5438	case SVM_EXIT_XCPT_SS:
5439	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5440	/* SVM_EXIT_XCPT_PF: */
5441	case SVM_EXIT_XCPT_15: /* Reserved. */
5442	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5443	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5444	case SVM_EXIT_XCPT_MC:
5445	case SVM_EXIT_XCPT_XF:
5446	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5447	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5448	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5449	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5450
5451	case SVM_EXIT_SWINT: VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5452	case SVM_EXIT_TR_READ: VMEXIT_CALL_RET(0, hmR0SvmExitTrRead(pVCpu, pSvmTransient));
5453	case SVM_EXIT_TR_WRITE: VMEXIT_CALL_RET(0, hmR0SvmExitTrWrite(pVCpu, pSvmTransient)); /* Also OS/2 TLB workaround. */
5454
5455	default:
5456	{
5457	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5458	pVCpu->hm.s.u32HMError = uExitCode;
5459	return VERR_SVM_UNKNOWN_EXIT;
5460	}
5461	}
5462	}
5463	}
5464	/* not reached */
5465	}
5466
5467
5468	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5469	*
5470	* The following few functions and associated structure contains the bloat
5471	* necessary for providing detailed debug events and dtrace probes as well as
5472	* reliable host side single stepping. This works on the principle of
5473	* "subclassing" the normal execution loop and workers. We replace the loop
5474	* method completely and override selected helpers to add necessary adjustments
5475	* to their core operation.
5476	*
5477	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5478	* any performance for debug and analysis features.
5479	*
5480	* @{
5481	*/
5482
5483	/**
5484	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
5485	* the debug run loop.
5486	*/
5487	typedef struct SVMRUNDBGSTATE
5488	{
5489	/** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5490	uint64_t bmInterceptInitial;
5491	/** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5492	uint32_t bmXcptInitial;
5493	/** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5494	uint16_t bmInterceptRdCRxInitial;
5495	/** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5496	uint16_t bmInterceptWrCRxInitial;
5497	/** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5498	uint16_t bmInterceptRdDRxInitial;
5499	/** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5500	uint16_t bmInterceptWrDRxInitial;
5501
5502	/** Whether we've actually modified the intercept control qword. */
5503	bool fModifiedInterceptCtrl : 1;
5504	/** Whether we've actually modified the exception bitmap. */
5505	bool fModifiedXcptBitmap : 1;
5506	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5507	bool fModifiedInterceptRdCRx : 1;
5508	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5509	bool fModifiedInterceptWrCRx : 1;
5510	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5511	bool fModifiedInterceptRdDRx : 1;
5512	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5513	bool fModifiedInterceptWrDRx : 1;
5514
5515	/** The CS we started executing with. */
5516	uint16_t uCsStart;
5517	/** The RIP we started executing at. This is for detecting that we stepped. */
5518	uint64_t uRipStart;
5519
5520	/** The sequence number of the Dtrace provider settings the state was
5521	* configured against. */
5522	uint32_t uDtraceSettingsSeqNo;
5523	/** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5524	uint32_t bmXcptExtra;
5525	/** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5526	uint64_t bmInterceptExtra;
5527	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5528	uint16_t bmInterceptRdCRxExtra;
5529	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5530	uint16_t bmInterceptWrCRxExtra;
5531	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5532	uint16_t bmInterceptRdDRxExtra;
5533	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5534	uint16_t bmInterceptWrDRxExtra;
5535	/** VM-exits to check (one bit per VM-exit). */
5536	uint32_t bmExitsToCheck[33];
5537	} SVMRUNDBGSTATE;
5538	AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5539	typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5540
5541
5542	/**
5543	* Initializes the SVMRUNDBGSTATE structure.
5544	*
5545	* @param pVCpu The cross context virtual CPU structure of the
5546	* calling EMT.
5547	* @param pSvmTransient The SVM-transient structure.
5548	* @param pDbgState The debug state to initialize.
5549	*/
5550	static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5551	{
5552	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5553	pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5554	pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5555	pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5556	pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5557	pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5558	pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5559
5560	pDbgState->fModifiedInterceptCtrl = false;
5561	pDbgState->fModifiedXcptBitmap = false;
5562	pDbgState->fModifiedInterceptRdCRx = false;
5563	pDbgState->fModifiedInterceptWrCRx = false;
5564	pDbgState->fModifiedInterceptRdDRx = false;
5565	pDbgState->fModifiedInterceptWrDRx = false;
5566
5567	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5568	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5569
5570	/* We don't really need to zero these. */
5571	pDbgState->bmInterceptExtra = 0;
5572	pDbgState->bmXcptExtra = 0;
5573	pDbgState->bmInterceptRdCRxExtra = 0;
5574	pDbgState->bmInterceptWrCRxExtra = 0;
5575	pDbgState->bmInterceptRdDRxExtra = 0;
5576	pDbgState->bmInterceptWrDRxExtra = 0;
5577	}
5578
5579
5580	/**
5581	* Updates the VMCB fields with changes requested by @a pDbgState.
5582	*
5583	* This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5584	* immediately before executing guest code, i.e. when interrupts are disabled.
5585	* We don't check status codes here as we cannot easily assert or return in the
5586	* latter case.
5587	*
5588	* @param pSvmTransient The SVM-transient structure.
5589	* @param pDbgState The debug state.
5590	*/
5591	static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5592	{
5593	/*
5594	* Ensure desired flags in VMCS control fields are set.
5595	*/
5596	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5597	#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5598	if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5599	{ \
5600	pVmcb->ctrl. a_VmcbCtrlField \|= (a_bmExtra); \
5601	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5602	Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5603	(a_fModified) = true; \
5604	} \
5605	} while (0)
5606	ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5607	ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5608	ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5609	ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5610	ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5611	ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5612	#undef ADD_EXTRA_INTERCEPTS
5613	}
5614
5615
5616	/**
5617	* Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5618	* for re-entry next time around.
5619	*
5620	* @returns Strict VBox status code (i.e. informational status codes too).
5621	* @param pSvmTransient The SVM-transient structure.
5622	* @param pDbgState The debug state.
5623	*/
5624	static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5625	{
5626	/*
5627	* Restore VM-exit control settings as we may not reenter this function the
5628	* next time around.
5629	*/
5630	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5631
5632	#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5633	if ((a_fModified)) \
5634	{ \
5635	pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5636	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5637	} \
5638	} while (0)
5639	RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5640	RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5641	RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5642	RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5643	RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5644	RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5645	#undef RESTORE_INTERCEPTS
5646	}
5647
5648
5649	/**
5650	* Configures VM-exit controls for current DBGF and DTrace settings.
5651	*
5652	* This updates @a pDbgState and the VMCB execution control fields (in the debug
5653	* state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5654	*
5655	* @param pVCpu The cross context virtual CPU structure.
5656	* @param pSvmTransient The SVM-transient structure. May update
5657	* fUpdatedTscOffsettingAndPreemptTimer.
5658	* @param pDbgState The debug state.
5659	*/
5660	static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5661	{
5662	/*
5663	* Take down the dtrace serial number so we can spot changes.
5664	*/
5665	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5666	ASMCompilerBarrier();
5667
5668	/*
5669	* Clear data members that we'll be rebuilding here.
5670	*/
5671	pDbgState->bmXcptExtra = 0;
5672	pDbgState->bmInterceptExtra = 0;
5673	pDbgState->bmInterceptRdCRxExtra = 0;
5674	pDbgState->bmInterceptWrCRxExtra = 0;
5675	pDbgState->bmInterceptRdDRxExtra = 0;
5676	pDbgState->bmInterceptWrDRxExtra = 0;
5677	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5678	pDbgState->bmExitsToCheck[i] = 0;
5679
5680	/*
5681	* Software interrupts (INT XXh)
5682	*/
5683	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5684	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5685	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
5686	{
5687	pDbgState->bmInterceptExtra \|= SVM_CTRL_INTERCEPT_INTN;
5688	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5689	}
5690
5691	/*
5692	* INT3 breakpoints - triggered by #BP exceptions.
5693	*/
5694	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
5695	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
5696
5697	/*
5698	* Exception bitmap and XCPT events+probes.
5699	*/
5700	#define SET_XCPT(a_iXcpt) do { \
5701	pDbgState->bmXcptExtra \|= RT_BIT_32(a_iXcpt); \
5702	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5703	} while (0)
5704
5705	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5706	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5707	SET_XCPT(iXcpt);
5708
5709	if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5710	if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5711	if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5712	if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5713	if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5714	if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5715	if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5716	if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5717	if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5718	if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5719	if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5720	if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5721	if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5722	if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5723	if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5724	if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5725	if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5726	if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5727
5728	#undef SET_XCPT
5729
5730	/*
5731	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5732	*
5733	* Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5734	* So, when adding/changing/removing please don't forget to update it.
5735	*
5736	* Some of the macros are picking up local variables to save horizontal space,
5737	* (being able to see it in a table is the lesser evil here).
5738	*/
5739	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5740	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5741	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5742	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5743	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5744	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5745	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5746	} else do { } while (0)
5747	#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5748	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5749	{ \
5750	(pDbgState)->bmInterceptExtra \|= (a_fInterceptCtrl); \
5751	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5752	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5753	} else do { } while (0)
5754
5755	/** @todo double check these */
5756	/** @todo Check what more AMD-V specific we can intercept. */
5757	//SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5758	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5759	SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5761	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5762	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5763	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5764	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5765	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5766	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5767	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5768	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5769	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5771
5772	SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5774	SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5775	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5776	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5777	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5778	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5779	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5780	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5781	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5782	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5783	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5784	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5785	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5786	SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5787	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5788
5789	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5790	pDbgState->bmInterceptRdCRxExtra = 0xffff;
5791	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5792	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5793
5794	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5795	pDbgState->bmInterceptWrCRxExtra = 0xffff;
5796	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5797	{
5798	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5799	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5800	}
5801
5802	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5803	pDbgState->bmInterceptRdDRxExtra = 0xffff;
5804	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5805	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5806
5807	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5808	pDbgState->bmInterceptWrDRxExtra = 0xffff;
5809	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5810	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5811
5812	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5813	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5814	SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5815	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5816	SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5817	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5818	if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5819	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5820	SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5821	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5822	SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5823	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5824	SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5825	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5826	SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5827	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5828	SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5829	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5830	SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5831	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5832	SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5833	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5834	SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5835	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5836	SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5837	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5838	SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5839	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5840	SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5841	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5842	SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5843	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5844
5845	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5846	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5847
5848	#undef IS_EITHER_ENABLED
5849	#undef SET_ONLY_XBM_IF_EITHER_EN
5850	#undef SET_INCP_XBM_IF_EITHER_EN
5851
5852	/*
5853	* Sanitize the control stuff.
5854	*/
5855	/** @todo filter out unsupported stuff? */
5856	if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5857	!= RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
5858	{
5859	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5860	/// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5861	RT_NOREF(pSvmTransient);
5862	}
5863
5864	Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5865	pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5866	pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5867	pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5868	pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5869	pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5870	pDbgState->bmExitsToCheck[0],
5871	pDbgState->bmExitsToCheck[1],
5872	pDbgState->bmExitsToCheck[2],
5873	pDbgState->bmExitsToCheck[3],
5874	pDbgState->bmExitsToCheck[4]));
5875	}
5876
5877
5878	/**
5879	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
5880	* appropriate.
5881	*
5882	* The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5883	* bitmap.
5884	*
5885	* @returns Strict VBox status code (i.e. informational status codes too).
5886	* @param pVCpu The cross context virtual CPU structure.
5887	* @param pSvmTransient The SVM-transient structure.
5888	* @param uExitCode The VM-exit code.
5889	*
5890	* @remarks The name of this function is displayed by dtrace, so keep it short
5891	* and to the point. No longer than 33 chars long, please.
5892	*/
5893	static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5894	{
5895	/*
5896	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5897	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
5898	*
5899	* Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5900	* does. Must add/change/remove both places. Same ordering, please.
5901	*
5902	* Added/removed events must also be reflected in the next section
5903	* where we dispatch dtrace events.
5904	*/
5905	bool fDtrace1 = false;
5906	bool fDtrace2 = false;
5907	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5908	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5909	uint64_t uEventArg = 0;
5910	#define SET_XCPT(a_XcptName) \
5911	do { \
5912	enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5913	fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5914	} while (0)
5915	#define SET_EXIT(a_EventSubName) \
5916	do { \
5917	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5918	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5919	} while (0)
5920	#define SET_BOTH(a_EventSubName) \
5921	do { \
5922	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5923	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5924	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5925	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5926	} while (0)
5927	switch (uExitCode)
5928	{
5929	case SVM_EXIT_SWINT:
5930	enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5931	fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5932	uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5933	break;
5934
5935	case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5936	case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5937	case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5938	case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5939	case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5940	case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5941	case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5942	case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5943	case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5944	case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5945	case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5946	case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5947	case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5948	case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5949	case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5950	case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5951	case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5952	case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5953
5954	case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5955	case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5956	case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5957	case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5958	case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5959	case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5960	case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5961	case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5962	case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5963	case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5964	case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5965	case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5966	case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5967	case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5968
5969	case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5970	case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5971	case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5972	case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5973	case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5974	case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5975	case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5976	case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5977	case SVM_EXIT_HLT: SET_BOTH(HALT); break;
5978	case SVM_EXIT_INVD: SET_BOTH(INVD); break;
5979	case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
5980	case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
5981	case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
5982	case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
5983	case SVM_EXIT_RSM: SET_BOTH(RSM); break;
5984
5985	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
5986	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
5987	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
5988	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
5989	SET_BOTH(CRX_READ);
5990	uEventArg = uExitCode - SVM_EXIT_READ_CR0;
5991	break;
5992	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
5993	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
5994	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
5995	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
5996	case SVM_EXIT_CR0_SEL_WRITE:
5997	SET_BOTH(CRX_WRITE);
5998	uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
5999	break;
6000	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6001	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6002	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6003	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6004	SET_BOTH(DRX_READ);
6005	uEventArg = uExitCode - SVM_EXIT_READ_DR0;
6006	break;
6007	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6008	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6009	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6010	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6011	SET_BOTH(DRX_WRITE);
6012	uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
6013	break;
6014	case SVM_EXIT_MSR:
6015	if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6016	SET_BOTH(WRMSR);
6017	else
6018	SET_BOTH(RDMSR);
6019	break;
6020	case SVM_EXIT_MWAIT_ARMED:
6021	case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6022	case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6023	case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6024	case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6025	case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6026	case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6027	case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6028	case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6029	case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6030	case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6031	case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6032	case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6033	case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6034
6035	case SVM_EXIT_SHUTDOWN:
6036	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6037	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6038	break;
6039
6040	default:
6041	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6042	break;
6043	}
6044	#undef SET_BOTH
6045	#undef SET_EXIT
6046
6047	/*
6048	* Dtrace tracepoints go first. We do them here at once so we don't
6049	* have to copy the guest state saving and stuff a few dozen times.
6050	* Down side is that we've got to repeat the switch, though this time
6051	* we use enmEvent since the probes are a subset of what DBGF does.
6052	*/
6053	if (fDtrace1 \|\| fDtrace2)
6054	{
6055	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6056	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6057	switch (enmEvent1)
6058	{
6059	/** @todo consider which extra parameters would be helpful for each probe. */
6060	case DBGFEVENT_END: break;
6061	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6062	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6063	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6064	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6065	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6066	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6067	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6068	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6069	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6070	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6071	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6072	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6073	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6074	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6075	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6076	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6077	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6078	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6079	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6080	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6081	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6082	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6083	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6084	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6085	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6086	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6087	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6088	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6089	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6090	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6091	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6092	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6093	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6094	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6095	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6096	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6097	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6098	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6099	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6100	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6101	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6102	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6103	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6104	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6105	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6106	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6107	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6108	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6109	case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6110	case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6111	case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6112	case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6113	case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6114	default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6115	}
6116	switch (enmEvent2)
6117	{
6118	/** @todo consider which extra parameters would be helpful for each probe. */
6119	case DBGFEVENT_END: break;
6120	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6121	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6122	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6123	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6124	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6125	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6126	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6127	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6128	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6129	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6130	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6131	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6132	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6133	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6134	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6135	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6136	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6137	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6138	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6139	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6140	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6141	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6142	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6143	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6144	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6145	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6146	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6147	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6148	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6149	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6150	case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6151	case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6152	case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6153	case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6154	case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6155	default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6156	}
6157	}
6158
6159	/*
6160	* Fire of the DBGF event, if enabled (our check here is just a quick one,
6161	* the DBGF call will do a full check).
6162	*
6163	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6164	* Note! If we have to events, we prioritize the first, i.e. the instruction
6165	* one, in order to avoid event nesting.
6166	*/
6167	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6168	VBOXSTRICTRC rcStrict;
6169	if ( enmEvent1 != DBGFEVENT_END
6170	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6171	{
6172	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6173	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6174	}
6175	else if ( enmEvent2 != DBGFEVENT_END
6176	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6177	{
6178	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6179	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6180	}
6181	else
6182	rcStrict = VINF_SUCCESS;
6183	return rcStrict;
6184	}
6185
6186
6187	/**
6188	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6189	* debug variant.
6190	*
6191	* @returns Strict VBox status code (informational status codes included).
6192	* @param pVCpu The cross context virtual CPU structure.
6193	* @param pSvmTransient Pointer to the SVM transient structure.
6194	* @param pDbgState The runtime debug state.
6195	*/
6196	static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6197	{
6198	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6199	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6200
6201	/*
6202	* Expensive (saves context) generic dtrace VM-exit probe.
6203	*/
6204	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6205	if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6206	{ /* more likely */ }
6207	else
6208	{
6209	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6210	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6211	}
6212
6213	/*
6214	* Check for single stepping event if we're stepping.
6215	*/
6216	if (pVCpu->hm.s.fSingleInstruction)
6217	{
6218	switch (uExitCode)
6219	{
6220	/* Various events: */
6221	case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6222	case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6223	case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6224	case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6225	case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6226	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6227	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6228	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6229	case SVM_EXIT_INTR:
6230	case SVM_EXIT_NMI:
6231	case SVM_EXIT_VINTR:
6232	case SVM_EXIT_NPF:
6233	case SVM_EXIT_AVIC_NOACCEL:
6234
6235	/* Instruction specific VM-exits: */
6236	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6237	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6238	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6239	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6240	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6241	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6242	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6243	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6244	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6245	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6246	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6247	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6248	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6249	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6250	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6251	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6252	case SVM_EXIT_CR0_SEL_WRITE:
6253	case SVM_EXIT_IDTR_READ:
6254	case SVM_EXIT_GDTR_READ:
6255	case SVM_EXIT_LDTR_READ:
6256	case SVM_EXIT_TR_READ:
6257	case SVM_EXIT_IDTR_WRITE:
6258	case SVM_EXIT_GDTR_WRITE:
6259	case SVM_EXIT_LDTR_WRITE:
6260	case SVM_EXIT_TR_WRITE:
6261	case SVM_EXIT_RDTSC:
6262	case SVM_EXIT_RDPMC:
6263	case SVM_EXIT_PUSHF:
6264	case SVM_EXIT_POPF:
6265	case SVM_EXIT_CPUID:
6266	case SVM_EXIT_RSM:
6267	case SVM_EXIT_IRET:
6268	case SVM_EXIT_SWINT:
6269	case SVM_EXIT_INVD:
6270	case SVM_EXIT_PAUSE:
6271	case SVM_EXIT_HLT:
6272	case SVM_EXIT_INVLPG:
6273	case SVM_EXIT_INVLPGA:
6274	case SVM_EXIT_IOIO:
6275	case SVM_EXIT_MSR:
6276	case SVM_EXIT_TASK_SWITCH:
6277	case SVM_EXIT_VMRUN:
6278	case SVM_EXIT_VMMCALL:
6279	case SVM_EXIT_VMLOAD:
6280	case SVM_EXIT_VMSAVE:
6281	case SVM_EXIT_STGI:
6282	case SVM_EXIT_CLGI:
6283	case SVM_EXIT_SKINIT:
6284	case SVM_EXIT_RDTSCP:
6285	case SVM_EXIT_ICEBP:
6286	case SVM_EXIT_WBINVD:
6287	case SVM_EXIT_MONITOR:
6288	case SVM_EXIT_MWAIT:
6289	case SVM_EXIT_MWAIT_ARMED:
6290	case SVM_EXIT_XSETBV:
6291	case SVM_EXIT_RDPRU:
6292	case SVM_EXIT_WRITE_EFER_TRAP:
6293	case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6294	case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6295	case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6296	case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6297	case SVM_EXIT_MCOMMIT:
6298	{
6299	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6300	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6301	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6302	{
6303	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6304	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6305	return VINF_EM_DBG_STEPPED;
6306	}
6307	break;
6308	}
6309
6310	/* Errors and unexpected events: */
6311	case SVM_EXIT_FERR_FREEZE:
6312	case SVM_EXIT_SHUTDOWN:
6313	case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6314	break;
6315
6316	case SVM_EXIT_SMI:
6317	case SVM_EXIT_INIT:
6318	default:
6319	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6320	break;
6321	}
6322	}
6323
6324	/*
6325	* Check for debugger event breakpoints and dtrace probes.
6326	*/
6327	if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6328	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6329	{
6330	VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6331	if (rcStrict != VINF_SUCCESS)
6332	{
6333	Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6334	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6335	return rcStrict;
6336	}
6337	}
6338
6339	/*
6340	* Normal processing.
6341	*/
6342	return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6343	}
6344
6345
6346	/**
6347	* Runs the guest code using AMD-V in single step mode.
6348	*
6349	* @returns Strict VBox status code.
6350	* @param pVCpu The cross context virtual CPU structure.
6351	* @param pcLoops Pointer to the number of executed loops.
6352	*/
6353	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6354	{
6355	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6356	Assert(pcLoops);
6357	Assert(*pcLoops <= cMaxResumeLoops);
6358
6359	SVMTRANSIENT SvmTransient;
6360	RT_ZERO(SvmTransient);
6361	SvmTransient.fUpdateTscOffsetting = true;
6362	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6363
6364	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6365
6366	/* Set HMCPU indicators. */
6367	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6368	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
6369	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6370	pVCpu->hmr0.s.fUsingDebugLoop = true;
6371
6372	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6373	SVMRUNDBGSTATE DbgState;
6374	hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6375	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6376
6377	/*
6378	* The loop.
6379	*/
6380	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6381	for (;;)
6382	{
6383	Assert(!HMR0SuspendPending());
6384	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6385	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6386	(unsigned)RTMpCpuId(), *pcLoops));
6387	bool fStepping = pVCpu->hm.s.fSingleInstruction;
6388
6389	/* Set up VM-execution controls the next two can respond to. */
6390	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6391
6392	/* Preparatory work for running nested-guest code, this may force us to return to
6393	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6394	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6395	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6396	if (rc != VINF_SUCCESS)
6397	break;
6398
6399	/*
6400	* No longjmps to ring-3 from this point on!!!
6401	*
6402	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6403	* better than a kernel panic. This also disables flushing of the R0-logger instance.
6404	*/
6405	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6406
6407	/* Override any obnoxious code in the above two calls. */
6408	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6409	#if 0
6410	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64\n",
6411	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6412	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u));
6413	#endif
6414
6415	/*
6416	* Finally execute guest code.
6417	*/
6418	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6419
6420	/* Restore any residual host-state and save any bits shared between host and guest
6421	into the guest-CPU state. Re-enables interrupts! */
6422	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6423	#if 0
6424	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64 exit=%d\n",
6425	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6426	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u, SvmTransient.u64ExitCode));
6427	#endif
6428
6429	if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6430	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6431	{ /* very likely*/ }
6432	else
6433	{
6434	if (rc == VINF_SUCCESS)
6435	rc = VERR_SVM_INVALID_GUEST_STATE;
6436	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6437	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6438	return rc;
6439	}
6440
6441	/* Handle the #VMEXIT. */
6442	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6443	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6444	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6445	rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6446	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6447	if (rc != VINF_SUCCESS)
6448	break;
6449	if (++(*pcLoops) >= cMaxResumeLoops)
6450	{
6451	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6452	rc = VINF_EM_RAW_INTERRUPT;
6453	break;
6454	}
6455
6456	/*
6457	* Stepping: Did the RIP change, if so, consider it a single step.
6458	* Otherwise, make sure one of the TFs gets set.
6459	*/
6460	if (fStepping)
6461	{
6462	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6463	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6464	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6465	{
6466	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6467	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6468	rc = VINF_EM_DBG_STEPPED;
6469	break;
6470	}
6471	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6472	}
6473
6474	/*
6475	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
6476	* Revert the state changes afterware so we can drop intercepts no longer needed.
6477	*/
6478	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6479	{
6480	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6481	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6482	}
6483	}
6484
6485	/*
6486	* Clear the X86_EFL_TF if necessary.
6487	*/
6488	if (pVCpu->hmr0.s.fClearTrapFlag)
6489	{
6490	pVCpu->hmr0.s.fClearTrapFlag = false;
6491	pCtx->eflags.Bits.u1TF = 0;
6492	}
6493
6494	/* Restore HMCPU indicators. */
6495	pVCpu->hmr0.s.fUsingDebugLoop = false;
6496	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6497	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6498
6499	/* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6500	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6501
6502	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6503	return rc;
6504	}
6505
6506	/** @} */
6507
6508	#undef VMEXIT_CALL_RET
6509
6510
6511	#ifdef VBOX_STRICT
6512	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6513	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6514	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6515
6516	# define HMSVM_ASSERT_PREEMPT_CPUID() \
6517	do \
6518	{ \
6519	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6520	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6521	} while (0)
6522
6523	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6524	do { \
6525	AssertPtr((a_pVCpu)); \
6526	AssertPtr((a_pSvmTransient)); \
6527	Assert(ASMIntAreEnabled()); \
6528	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6529	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6530	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6531	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6532	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6533	HMSVM_ASSERT_PREEMPT_CPUID(); \
6534	} while (0)
6535	#else
6536	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6537	do { \
6538	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6539	} while (0)
6540	#endif
6541
6542
6543	/**
6544	* Gets the IEM exception flags for the specified SVM event.
6545	*
6546	* @returns The IEM exception flags.
6547	* @param pEvent Pointer to the SVM event.
6548	*
6549	* @remarks This function currently only constructs flags required for
6550	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6551	* and CR2 aspects of an exception are not included).
6552	*/
6553	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6554	{
6555	uint8_t const uEventType = pEvent->n.u3Type;
6556	uint32_t fIemXcptFlags;
6557	switch (uEventType)
6558	{
6559	case SVM_EVENT_EXCEPTION:
6560	/*
6561	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6562	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6563	*/
6564	if (pEvent->n.u8Vector == X86_XCPT_BP)
6565	{
6566	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6567	break;
6568	}
6569	if (pEvent->n.u8Vector == X86_XCPT_OF)
6570	{
6571	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6572	break;
6573	}
6574	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6575	RT_FALL_THRU();
6576	case SVM_EVENT_NMI:
6577	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6578	break;
6579
6580	case SVM_EVENT_EXTERNAL_IRQ:
6581	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6582	break;
6583
6584	case SVM_EVENT_SOFTWARE_INT:
6585	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6586	break;
6587
6588	default:
6589	fIemXcptFlags = 0;
6590	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6591	break;
6592	}
6593	return fIemXcptFlags;
6594	}
6595
6596
6597	/**
6598	* Handle a condition that occurred while delivering an event through the guest
6599	* IDT.
6600	*
6601	* @returns VBox status code (informational error codes included).
6602	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6603	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6604	* continue execution of the guest which will delivery the \#DF.
6605	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6606	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6607	*
6608	* @param pVCpu The cross context virtual CPU structure.
6609	* @param pSvmTransient Pointer to the SVM transient structure.
6610	*
6611	* @remarks No-long-jump zone!!!
6612	*/
6613	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6614	{
6615	/** @todo r=bird: Looks like this is called on many exits and we start by
6616	* loading CR2 on the offchance that we actually have work to do here.
6617	*
6618	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY can surely check
6619	* pVmcb->ctrl.ExitIntInfo.n.u1Valid, can't it?
6620	*
6621	* Also, what's the deal with hmR0SvmGetCurrentVmcb() vs pSvmTransient->pVmcb?
6622	*/
6623	int rc = VINF_SUCCESS;
6624	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6625	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6626
6627	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6628	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6629	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6630
6631	/*
6632	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6633	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6634	*
6635	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6636	*/
6637	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6638	{
6639	IEMXCPTRAISE enmRaise;
6640	IEMXCPTRAISEINFO fRaiseInfo;
6641	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6642	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6643	if (fExitIsHwXcpt)
6644	{
6645	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6646	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6647	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6648	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6649	}
6650	else
6651	{
6652	/*
6653	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6654	* then we end up here.
6655	*
6656	* If the event was:
6657	* - a software interrupt, we can re-execute the instruction which will
6658	* regenerate the event.
6659	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6660	* - a hardware exception or external interrupt, we re-inject it.
6661	*/
6662	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6663	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6664	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6665	else
6666	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6667	}
6668
6669	switch (enmRaise)
6670	{
6671	case IEMXCPTRAISE_CURRENT_XCPT:
6672	case IEMXCPTRAISE_PREV_EVENT:
6673	{
6674	/* For software interrupts, we shall re-execute the instruction. */
6675	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6676	{
6677	RTGCUINTPTR GCPtrFaultAddress = 0;
6678
6679	/* If we are re-injecting an NMI, clear NMI blocking. */
6680	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6681	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6682
6683	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6684	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6685	{
6686	pSvmTransient->fVectoringPF = true;
6687	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6688	pVCpu->cpum.GstCtx.cr2));
6689	}
6690	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6691	&& uIdtVector == X86_XCPT_PF)
6692	{
6693	/*
6694	* If the previous exception was a #PF, we need to recover the CR2 value.
6695	* This can't happen with shadow paging.
6696	*/
6697	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6698	}
6699
6700	/*
6701	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6702	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6703	*/
6704	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6705	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6706	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6707
6708	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6709	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6710	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6711	}
6712	break;
6713	}
6714
6715	case IEMXCPTRAISE_REEXEC_INSTR:
6716	{
6717	Assert(rc == VINF_SUCCESS);
6718	break;
6719	}
6720
6721	case IEMXCPTRAISE_DOUBLE_FAULT:
6722	{
6723	/*
6724	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6725	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6726	* converted into a #DF.
6727	*/
6728	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6729	{
6730	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6731	pSvmTransient->fVectoringDoublePF = true;
6732	Assert(rc == VINF_SUCCESS);
6733	}
6734	else
6735	{
6736	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6737	hmR0SvmSetPendingXcptDF(pVCpu);
6738	rc = VINF_HM_DOUBLE_FAULT;
6739	}
6740	break;
6741	}
6742
6743	case IEMXCPTRAISE_TRIPLE_FAULT:
6744	{
6745	rc = VINF_EM_RESET;
6746	break;
6747	}
6748
6749	case IEMXCPTRAISE_CPU_HANG:
6750	{
6751	rc = VERR_EM_GUEST_CPU_HANG;
6752	break;
6753	}
6754
6755	default:
6756	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6757	}
6758	}
6759	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6760	return rc;
6761	}
6762
6763
6764	/**
6765	* Advances the guest RIP by the number of bytes specified in @a cb.
6766	*
6767	* @param pVCpu The cross context virtual CPU structure.
6768	* @param cb RIP increment value in bytes.
6769	*/
6770	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6771	{
6772	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6773	pCtx->rip += cb;
6774
6775	/* Update interrupt shadow. */
6776	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
6777	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
6778	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6779	}
6780
6781
6782	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6783	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6784	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6785
6786	/** @name \#VMEXIT handlers.
6787	* @{
6788	*/
6789
6790	/**
6791	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6792	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6793	*/
6794	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6795	{
6796	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6797
6798	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6799	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6800	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6801	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6802
6803	/*
6804	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6805	* signal -before- the timer fires if the current interrupt is our own timer or a some
6806	* other host interrupt. We also cannot examine what interrupt it is until the host
6807	* actually take the interrupt.
6808	*
6809	* Going back to executing guest code here unconditionally causes random scheduling
6810	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6811	*/
6812	return VINF_EM_RAW_INTERRUPT;
6813	}
6814
6815
6816	/**
6817	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6818	*/
6819	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6820	{
6821	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6822
6823	VBOXSTRICTRC rcStrict;
6824	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6825	if (fSupportsNextRipSave)
6826	{
6827	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6828	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6829	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6830	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6831	}
6832	else
6833	{
6834	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6835	rcStrict = IEMExecOne(pVCpu);
6836	}
6837
6838	if (rcStrict == VINF_IEM_RAISED_XCPT)
6839	{
6840	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6841	rcStrict = VINF_SUCCESS;
6842	}
6843	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6844	return rcStrict;
6845	}
6846
6847
6848	/**
6849	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6850	*/
6851	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6852	{
6853	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6854
6855	VBOXSTRICTRC rcStrict;
6856	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6857	if (fSupportsNextRipSave)
6858	{
6859	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6860	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6861	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6862	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6863	}
6864	else
6865	{
6866	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6867	rcStrict = IEMExecOne(pVCpu);
6868	}
6869
6870	if (rcStrict == VINF_IEM_RAISED_XCPT)
6871	{
6872	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6873	rcStrict = VINF_SUCCESS;
6874	}
6875	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6876	return rcStrict;
6877	}
6878
6879
6880	/**
6881	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6882	*/
6883	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6884	{
6885	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6886
6887	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6888	VBOXSTRICTRC rcStrict;
6889	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6890	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6891	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6892	if (!pExitRec)
6893	{
6894	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6895	if (fSupportsNextRipSave)
6896	{
6897	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6898	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6899	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6900	}
6901	else
6902	{
6903	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6904	rcStrict = IEMExecOne(pVCpu);
6905	}
6906
6907	if (rcStrict == VINF_IEM_RAISED_XCPT)
6908	{
6909	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6910	rcStrict = VINF_SUCCESS;
6911	}
6912	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6913	}
6914	else
6915	{
6916	/*
6917	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6918	*/
6919	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6920
6921	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6922	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6923
6924	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6925
6926	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6927	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6928	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6929	}
6930	return rcStrict;
6931	}
6932
6933
6934	/**
6935	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6936	*/
6937	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6938	{
6939	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6940
6941	VBOXSTRICTRC rcStrict;
6942	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6943	if (fSupportsNextRipSave)
6944	{
6945	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6946	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6947	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6948	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6949	}
6950	else
6951	{
6952	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6953	rcStrict = IEMExecOne(pVCpu);
6954	}
6955
6956	if (rcStrict == VINF_SUCCESS)
6957	pSvmTransient->fUpdateTscOffsetting = true;
6958	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6959	{
6960	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6961	rcStrict = VINF_SUCCESS;
6962	}
6963	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6964	return rcStrict;
6965	}
6966
6967
6968	/**
6969	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6970	*/
6971	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6972	{
6973	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6974
6975	VBOXSTRICTRC rcStrict;
6976	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6977	if (fSupportsNextRipSave)
6978	{
6979	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
6980	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6981	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6982	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
6983	}
6984	else
6985	{
6986	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6987	rcStrict = IEMExecOne(pVCpu);
6988	}
6989
6990	if (rcStrict == VINF_SUCCESS)
6991	pSvmTransient->fUpdateTscOffsetting = true;
6992	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6993	{
6994	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6995	rcStrict = VINF_SUCCESS;
6996	}
6997	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6998	return rcStrict;
6999	}
7000
7001
7002	/**
7003	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
7004	*/
7005	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7006	{
7007	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7008
7009	VBOXSTRICTRC rcStrict;
7010	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7011	if (fSupportsNextRipSave)
7012	{
7013	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
7014	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7015	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7016	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
7017	}
7018	else
7019	{
7020	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7021	rcStrict = IEMExecOne(pVCpu);
7022	}
7023
7024	if (rcStrict == VINF_IEM_RAISED_XCPT)
7025	{
7026	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7027	rcStrict = VINF_SUCCESS;
7028	}
7029	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7030	return rcStrict;
7031	}
7032
7033
7034	/**
7035	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7036	*/
7037	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7038	{
7039	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7040	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7041
7042	VBOXSTRICTRC rcStrict;
7043	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7044	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7045	if ( fSupportsDecodeAssists
7046	&& fSupportsNextRipSave)
7047	{
7048	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7049	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7050	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7051	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7052	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7053	}
7054	else
7055	{
7056	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7057	rcStrict = IEMExecOne(pVCpu);
7058	}
7059
7060	if (rcStrict == VINF_IEM_RAISED_XCPT)
7061	{
7062	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7063	rcStrict = VINF_SUCCESS;
7064	}
7065	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7066	return VBOXSTRICTRC_VAL(rcStrict);
7067	}
7068
7069
7070	/**
7071	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7072	*/
7073	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7074	{
7075	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7076
7077	VBOXSTRICTRC rcStrict;
7078	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7079	if (fSupportsNextRipSave)
7080	{
7081	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7082	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7083	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7084	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7085	}
7086	else
7087	{
7088	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7089	rcStrict = IEMExecOne(pVCpu);
7090	}
7091
7092	if ( rcStrict == VINF_EM_HALT
7093	\|\| rcStrict == VINF_SUCCESS)
7094	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7095	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7096	{
7097	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7098	rcStrict = VINF_SUCCESS;
7099	}
7100	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7101	if (rcStrict != VINF_SUCCESS)
7102	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7103	return VBOXSTRICTRC_VAL(rcStrict);;
7104	}
7105
7106
7107	/**
7108	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7109	*/
7110	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7111	{
7112	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7113
7114	/*
7115	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7116	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
7117	* segment override prefix or other prefixes might be used, in which case we fallback to
7118	* IEMExecOne() to figure out.
7119	*/
7120	VBOXSTRICTRC rcStrict;
7121	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7122	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7123	if (cbInstr)
7124	{
7125	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7126	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7127	}
7128	else
7129	{
7130	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7131	rcStrict = IEMExecOne(pVCpu);
7132	}
7133
7134	if (rcStrict == VINF_IEM_RAISED_XCPT)
7135	{
7136	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7137	rcStrict = VINF_SUCCESS;
7138	}
7139	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7140	return rcStrict;
7141	}
7142
7143
7144	/**
7145	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7146	*/
7147	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7148	{
7149	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7150
7151	VBOXSTRICTRC rcStrict;
7152	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7153	if (fSupportsNextRipSave)
7154	{
7155	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7156	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7157	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7158	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7159	}
7160	else
7161	{
7162	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7163	rcStrict = IEMExecOne(pVCpu);
7164	}
7165
7166	if ( rcStrict == VINF_EM_HALT
7167	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7168	rcStrict = VINF_SUCCESS;
7169	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7170	{
7171	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7172	rcStrict = VINF_SUCCESS;
7173	}
7174	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7175	return rcStrict;
7176	}
7177
7178
7179	/**
7180	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7181	* \#VMEXIT.
7182	*/
7183	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7184	{
7185	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7186	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7187	return VINF_EM_RESET;
7188	}
7189
7190
7191	/**
7192	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7193	*/
7194	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7195	{
7196	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7197	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7198	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7199	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7200	RT_NOREF(pVmcb);
7201	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7202	return VERR_SVM_UNEXPECTED_EXIT;
7203	}
7204
7205
7206	/**
7207	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7208	*/
7209	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7210	{
7211	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7212
7213	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7214	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7215	#ifdef VBOX_WITH_STATISTICS
7216	switch (pSvmTransient->u64ExitCode)
7217	{
7218	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7219	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7220	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7221	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7222	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7223	}
7224	#endif
7225
7226	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7227	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7228	if ( fSupportsDecodeAssists
7229	&& fSupportsNextRipSave)
7230	{
7231	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7232	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7233	if (fMovCRx)
7234	{
7235	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
7236	\| CPUMCTX_EXTRN_APIC_TPR);
7237	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7238	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7239	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7240	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7241	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7242	return VBOXSTRICTRC_VAL(rcStrict);
7243	}
7244	/* else: SMSW instruction, fall back below to IEM for this. */
7245	}
7246
7247	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7248	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7249	AssertMsg( rcStrict == VINF_SUCCESS
7250	\|\| rcStrict == VINF_PGM_SYNC_CR3
7251	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
7252	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7253	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7254	if (rcStrict == VINF_IEM_RAISED_XCPT)
7255	{
7256	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7257	rcStrict = VINF_SUCCESS;
7258	}
7259	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7260	return rcStrict;
7261	}
7262
7263
7264	/**
7265	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7266	*/
7267	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7268	{
7269	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7270
7271	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7272	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7273	Assert(iCrReg <= 15);
7274
7275	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7276	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7277	bool fDecodedInstr = false;
7278	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7279	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7280	if ( fSupportsDecodeAssists
7281	&& fSupportsNextRipSave)
7282	{
7283	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7284	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7285	if (fMovCRx)
7286	{
7287	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
7288	\| CPUMCTX_EXTRN_APIC_TPR);
7289	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7290	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7291	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7292	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7293	fDecodedInstr = true;
7294	}
7295	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7296	}
7297
7298	if (!fDecodedInstr)
7299	{
7300	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7301	Log4Func(("iCrReg=%#x\n", iCrReg));
7302	rcStrict = IEMExecOne(pVCpu);
7303	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7304	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7305	rcStrict = VERR_EM_INTERPRETER;
7306	}
7307
7308	if (rcStrict == VINF_SUCCESS)
7309	{
7310	switch (iCrReg)
7311	{
7312	case 0:
7313	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7315	break;
7316
7317	case 2:
7318	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7319	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7320	break;
7321
7322	case 3:
7323	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7325	break;
7326
7327	case 4:
7328	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7329	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7330	break;
7331
7332	case 8:
7333	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7334	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7335	break;
7336
7337	default:
7338	{
7339	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7340	pSvmTransient->u64ExitCode, iCrReg));
7341	break;
7342	}
7343	}
7344	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7345	}
7346	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7347	{
7348	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7349	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7350	rcStrict = VINF_SUCCESS;
7351	}
7352	else
7353	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
7354	return rcStrict;
7355	}
7356
7357
7358	/**
7359	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7360	*
7361	* @returns Strict VBox status code.
7362	* @param pVCpu The cross context virtual CPU structure.
7363	* @param pVmcb Pointer to the VM control block.
7364	*/
7365	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7366	{
7367	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7368	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7369
7370	VBOXSTRICTRC rcStrict;
7371	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7372	if (fSupportsNextRipSave)
7373	{
7374	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7375	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7376	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7377	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7378	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7379	}
7380	else
7381	{
7382	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7383	rcStrict = IEMExecOne(pVCpu);
7384	}
7385
7386	AssertMsg( rcStrict == VINF_SUCCESS
7387	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7388	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
7389	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7390
7391	if (rcStrict == VINF_IEM_RAISED_XCPT)
7392	{
7393	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7394	rcStrict = VINF_SUCCESS;
7395	}
7396	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7397	return rcStrict;
7398	}
7399
7400
7401	/**
7402	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7403	*
7404	* @returns Strict VBox status code.
7405	* @param pVCpu The cross context virtual CPU structure.
7406	* @param pVmcb Pointer to the VM control block.
7407	* @param pSvmTransient Pointer to the SVM-transient structure.
7408	*/
7409	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7410	{
7411	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7412	uint32_t const idMsr = pCtx->ecx;
7413	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7414	Log4Func(("idMsr=%#RX32\n", idMsr));
7415
7416	/*
7417	* Handle TPR patching MSR writes.
7418	* We utilitize the LSTAR MSR for patching.
7419	*/
7420	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7421	if ( idMsr == MSR_K8_LSTAR
7422	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7423	{
7424	unsigned cbInstr;
7425	if (fSupportsNextRipSave)
7426	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7427	else
7428	{
7429	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
7430	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7431	if ( rc == VINF_SUCCESS
7432	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
7433	Assert(cbInstr > 0);
7434	else
7435	cbInstr = 0;
7436	}
7437
7438	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7439	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7440	{
7441	int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
7442	AssertRCReturn(rc, rc);
7443	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7444	}
7445
7446	int rc = VINF_SUCCESS;
7447	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7448	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7449	return rc;
7450	}
7451
7452	/*
7453	* Handle regular MSR writes.
7454	*/
7455	VBOXSTRICTRC rcStrict;
7456	if (fSupportsNextRipSave)
7457	{
7458	/** @todo Optimize this: We don't need to get much of the MSR state here
7459	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7460	* clear the applicable extern flags. */
7461	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7462	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7463	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7464	}
7465	else
7466	{
7467	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7468	rcStrict = IEMExecOne(pVCpu);
7469	}
7470
7471	AssertMsg( rcStrict == VINF_SUCCESS
7472	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7473	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
7474	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7475
7476	if (rcStrict == VINF_SUCCESS)
7477	{
7478	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7479	if ( idMsr >= MSR_IA32_X2APIC_START
7480	&& idMsr <= MSR_IA32_X2APIC_END)
7481	{
7482	/*
7483	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7484	* When full APIC register virtualization is implemented we'll have to make sure
7485	* APIC state is saved from the VMCB before IEM changes it.
7486	*/
7487	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7488	}
7489	else
7490	{
7491	switch (idMsr)
7492	{
7493	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7494	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7495	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7496	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7497	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7498	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7499	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7500	}
7501	}
7502	}
7503	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7504	{
7505	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7506	rcStrict = VINF_SUCCESS;
7507	}
7508	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7509	return rcStrict;
7510	}
7511
7512
7513	/**
7514	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7515	* \#VMEXIT.
7516	*/
7517	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7518	{
7519	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7520
7521	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7522	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7523	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7524
7525	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7526	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7527	}
7528
7529
7530	/**
7531	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7532	*/
7533	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7534	{
7535	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7536	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7537
7538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7539
7540	/** @todo Stepping with nested-guest. */
7541	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7542	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7543	{
7544	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7545	if (pSvmTransient->fWasGuestDebugStateActive)
7546	{
7547	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7548	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7549	return VERR_SVM_UNEXPECTED_EXIT;
7550	}
7551
7552	/*
7553	* Lazy DR0-3 loading.
7554	*/
7555	if (!pSvmTransient->fWasHyperDebugStateActive)
7556	{
7557	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7558	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7559
7560	/* Don't intercept DRx read and writes. */
7561	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7562	pVmcb->ctrl.u16InterceptRdDRx = 0;
7563	pVmcb->ctrl.u16InterceptWrDRx = 0;
7564	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7565
7566	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7567	VMMRZCallRing3Disable(pVCpu);
7568	HM_DISABLE_PREEMPT(pVCpu);
7569
7570	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7571	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7572	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7573
7574	HM_RESTORE_PREEMPT();
7575	VMMRZCallRing3Enable(pVCpu);
7576
7577	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7578	return VINF_SUCCESS;
7579	}
7580	}
7581
7582	/*
7583	* Interpret the read/writing of DRx.
7584	*/
7585	/** @todo Decode assist. */
7586	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
7587	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7588	if (RT_LIKELY(rc == VINF_SUCCESS))
7589	{
7590	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7591	/** @todo CPUM should set this flag! */
7592	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7593	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7594	}
7595	else
7596	Assert(rc == VERR_EM_INTERPRETER);
7597	return rc;
7598	}
7599
7600
7601	/**
7602	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7603	*/
7604	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7605	{
7606	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7607	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7608	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7610	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7611	return rc;
7612	}
7613
7614
7615	/**
7616	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7617	*/
7618	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7619	{
7620	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7621	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7622
7623	/** @todo decode assists... */
7624	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7625	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7626	{
7627	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7628	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7629	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7630	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7631	{
7632	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7633	hmR0SvmUpdateVmRunFunction(pVCpu);
7634	}
7635	}
7636	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7637	{
7638	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7639	rcStrict = VINF_SUCCESS;
7640	}
7641	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7642	return rcStrict;
7643	}
7644
7645
7646	/**
7647	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7648	*/
7649	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7650	{
7651	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7652	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
7653
7654	/* I/O operation lookup arrays. */
7655	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7656	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7657	the result (in AL/AX/EAX). */
7658	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7659	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7660	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7661
7662	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7663
7664	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7665	SVMIOIOEXITINFO IoExitInfo;
7666	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7667	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7668	uint32_t cbValue = s_aIOSize[uIOWidth];
7669	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7670
7671	if (RT_UNLIKELY(!cbValue))
7672	{
7673	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7674	return VERR_EM_INTERPRETER;
7675	}
7676
7677	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
7678	VBOXSTRICTRC rcStrict;
7679	PCEMEXITREC pExitRec = NULL;
7680	if ( !pVCpu->hm.s.fSingleInstruction
7681	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7682	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7683	!IoExitInfo.n.u1Str
7684	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7685	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7686	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7687	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7688	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7689	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7690	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7691	if (!pExitRec)
7692	{
7693	bool fUpdateRipAlready = false;
7694	if (IoExitInfo.n.u1Str)
7695	{
7696	/* INS/OUTS - I/O String instruction. */
7697	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7698	* in EXITINFO1? Investigate once this thing is up and running. */
7699	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7700	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7701	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7702	static IEMMODE const s_aenmAddrMode[8] =
7703	{
7704	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7705	};
7706	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7707	if (enmAddrMode != (IEMMODE)-1)
7708	{
7709	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7710	if (cbInstr <= 15 && cbInstr >= 1)
7711	{
7712	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7713	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7714	{
7715	/* Don't know exactly how to detect whether u3Seg is valid, currently
7716	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7717	2384 Opterons when only checking NRIP. */
7718	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7719	if ( fSupportsNextRipSave
7720	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7721	{
7722	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7723	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7724	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7725	IoExitInfo.n.u3Seg, true /fIoChecked/);
7726	}
7727	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7728	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7729	X86_SREG_DS, true /fIoChecked/);
7730	else
7731	rcStrict = IEMExecOne(pVCpu);
7732	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7733	}
7734	else
7735	{
7736	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7737	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7738	true /fIoChecked/);
7739	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7740	}
7741	}
7742	else
7743	{
7744	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7745	rcStrict = IEMExecOne(pVCpu);
7746	}
7747	}
7748	else
7749	{
7750	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7751	rcStrict = IEMExecOne(pVCpu);
7752	}
7753	fUpdateRipAlready = true;
7754	if (rcStrict == VINF_IEM_RAISED_XCPT)
7755	{
7756	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7757	rcStrict = VINF_SUCCESS;
7758	}
7759	}
7760	else
7761	{
7762	/* IN/OUT - I/O instruction. */
7763	Assert(!IoExitInfo.n.u1Rep);
7764
7765	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7766	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7767	{
7768	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7769	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7770	&& !pCtx->eflags.Bits.u1TF)
7771	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7772	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7773	}
7774	else
7775	{
7776	uint32_t u32Val = 0;
7777	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7778	if (IOM_SUCCESS(rcStrict))
7779	{
7780	/* Save result of I/O IN instr. in AL/AX/EAX. */
7781	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7782	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7783	}
7784	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7785	&& !pCtx->eflags.Bits.u1TF)
7786	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7787
7788	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7789	}
7790	}
7791
7792	if (IOM_SUCCESS(rcStrict))
7793	{
7794	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7795	if (!fUpdateRipAlready)
7796	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7797
7798	/*
7799	* If any I/O breakpoints are armed, we need to check if one triggered
7800	* and take appropriate action.
7801	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7802	*/
7803	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7804	* execution engines about whether hyper BPs and such are pending. */
7805	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7806	uint32_t const uDr7 = pCtx->dr[7];
7807	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7808	&& X86_DR7_ANY_RW_IO(uDr7)
7809	&& (pCtx->cr4 & X86_CR4_DE))
7810	\|\| DBGFBpIsHwIoArmed(pVM)))
7811	{
7812	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7813	VMMRZCallRing3Disable(pVCpu);
7814	HM_DISABLE_PREEMPT(pVCpu);
7815
7816	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7817	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7818
7819	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7820	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7821	{
7822	/* Raise #DB. */
7823	pVmcb->guest.u64DR6 = pCtx->dr[6];
7824	pVmcb->guest.u64DR7 = pCtx->dr[7];
7825	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7826	hmR0SvmSetPendingXcptDB(pVCpu);
7827	}
7828	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7829	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7830	else if ( rcStrict2 != VINF_SUCCESS
7831	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7832	rcStrict = rcStrict2;
7833	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7834
7835	HM_RESTORE_PREEMPT();
7836	VMMRZCallRing3Enable(pVCpu);
7837	}
7838
7839	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7840	}
7841	#ifdef VBOX_STRICT
7842	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7843	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7844	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7845	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7846	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7847	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7848	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7849	else
7850	{
7851	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7852	* statuses, that the VMM device and some others may return. See
7853	* IOM_SUCCESS() for guidance. */
7854	AssertMsg( RT_FAILURE(rcStrict)
7855	\|\| rcStrict == VINF_SUCCESS
7856	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7857	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7858	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7859	\|\| rcStrict == VINF_EM_DBG_STEPPED
7860	\|\| rcStrict == VINF_EM_RAW_TO_R3
7861	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7862	}
7863	#endif
7864	}
7865	else
7866	{
7867	/*
7868	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7869	*/
7870	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7871	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7872	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7873	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7874	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7875	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7876	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7877
7878	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7879	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7880
7881	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7882	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7883	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7884	}
7885	return rcStrict;
7886	}
7887
7888
7889	/**
7890	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7891	*/
7892	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7893	{
7894	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7895	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7896	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7897
7898	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7899	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7900	Assert(pVM->hmr0.s.fNestedPaging);
7901
7902	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7903	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7904	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7905	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7906	thus intentionally not copied into u32ErrCode. */
7907
7908	Log4Func(("#NPF at CS:RIP=%04x:%RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7909	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7910
7911	/*
7912	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7913	*/
7914	if ( pVM->hm.s.fTprPatchingAllowed
7915	&& (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7916	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7917	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7918	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7919	&& !CPUMIsGuestInLongModeEx(pCtx)
7920	&& !CPUMGetGuestCPL(pVCpu)
7921	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7922	{
7923	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7924	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7925
7926	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7927	{
7928	/* Only attempt to patch the instruction once. */
7929	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7930	if (!pPatch)
7931	return VINF_EM_HM_PATCH_TPR_INSTR;
7932	}
7933	}
7934
7935	/*
7936	* Determine the nested paging mode.
7937	*/
7938	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7939	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7940
7941	/*
7942	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7943	*/
7944	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7945	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7946	{
7947	/*
7948	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7949	* injecting the original pending event would most likely cause the same MMIO #NPF.
7950	*/
7951	if (pVCpu->hm.s.Event.fPending)
7952	{
7953	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7954	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7955	}
7956
7957	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7958	VBOXSTRICTRC rcStrict;
7959	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7960	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7961	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7962	if (!pExitRec)
7963	{
7964
7965	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
7966	u32ErrCode);
7967
7968	/*
7969	* If we succeed, resume guest execution.
7970	*
7971	* If we fail in interpreting the instruction because we couldn't get the guest
7972	* physical address of the page containing the instruction via the guest's page
7973	* tables (we would invalidate the guest page in the host TLB), resume execution
7974	* which would cause a guest page fault to let the guest handle this weird case.
7975	*
7976	* See @bugref{6043}.
7977	*/
7978	if ( rcStrict == VINF_SUCCESS
7979	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7980	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
7981	{
7982	/* Successfully handled MMIO operation. */
7983	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7984	rcStrict = VINF_SUCCESS;
7985	}
7986	}
7987	else
7988	{
7989	/*
7990	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7991	*/
7992	Assert(pCtx == &pVCpu->cpum.GstCtx);
7993	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7994	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7995	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
7996
7997	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7998	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7999
8000	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8001	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8002	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8003	}
8004	return rcStrict;
8005	}
8006
8007	/*
8008	* Nested page-fault.
8009	*/
8010	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
8011	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
8012	TRPMResetTrap(pVCpu);
8013
8014	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
8015
8016	/*
8017	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
8018	*/
8019	if ( rc == VINF_SUCCESS
8020	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8021	\|\| rc == VERR_PAGE_NOT_PRESENT)
8022	{
8023	/* We've successfully synced our shadow page tables. */
8024	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8025	rc = VINF_SUCCESS;
8026	}
8027
8028	/*
8029	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
8030	* re-inject the original event.
8031	*/
8032	if (pVCpu->hm.s.Event.fPending)
8033	{
8034	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8035
8036	/*
8037	* If the #NPF handler requested emulation of the instruction, ignore it.
8038	* We need to re-inject the original event so as to not lose it.
8039	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8040	* LiveCD is broken for other reasons).
8041	*/
8042	if (rc == VINF_EM_RAW_EMULATE_INSTR)
8043	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8044	}
8045
8046	return rc;
8047	}
8048
8049
8050	/**
8051	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8052	* \#VMEXIT.
8053	*/
8054	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8055	{
8056	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8057	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8058
8059	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8060	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8061	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8062
8063	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8064	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8065	return VINF_SUCCESS;
8066	}
8067
8068
8069	/**
8070	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8071	* \#VMEXIT.
8072	*/
8073	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8074	{
8075	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8076	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8077
8078	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8079	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8080	#endif
8081
8082	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
8083	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8084	{
8085	/*
8086	* AMD-V provides us with the exception which caused the TS; we collect
8087	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8088	*/
8089	Log4Func(("TS occurred during event delivery\n"));
8090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8091	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8092	}
8093
8094	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8095	* emulation. */
8096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8097	return VERR_EM_INTERPRETER;
8098	}
8099
8100
8101	/**
8102	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8103	*/
8104	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8105	{
8106	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8107	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8108
8109	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8110	if (pVM->hm.s.fTprPatchingAllowed)
8111	{
8112	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8113	if (rc != VERR_NOT_FOUND)
8114	{
8115	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8116	return rc;
8117	}
8118	}
8119
8120	if (EMAreHypercallInstructionsEnabled(pVCpu))
8121	{
8122	unsigned cbInstr;
8123	if (hmR0SvmSupportsNextRipSave(pVCpu))
8124	{
8125	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8126	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8127	}
8128	else
8129	{
8130	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8131	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8132	if ( rc == VINF_SUCCESS
8133	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
8134	Assert(cbInstr > 0);
8135	else
8136	cbInstr = 0;
8137	}
8138
8139	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8140	if (RT_SUCCESS(rcStrict))
8141	{
8142	/* Only update the RIP if we're continuing guest execution and not in the case
8143	of say VINF_GIM_R3_HYPERCALL. */
8144	if (rcStrict == VINF_SUCCESS)
8145	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8146
8147	return VBOXSTRICTRC_VAL(rcStrict);
8148	}
8149	else
8150	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8151	}
8152
8153	hmR0SvmSetPendingXcptUD(pVCpu);
8154	return VINF_SUCCESS;
8155	}
8156
8157
8158	/**
8159	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8160	*/
8161	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8162	{
8163	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8164
8165	unsigned cbInstr;
8166	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8167	if (fSupportsNextRipSave)
8168	{
8169	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8170	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8171	}
8172	else
8173	{
8174	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8175	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8176	if ( rc == VINF_SUCCESS
8177	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
8178	Assert(cbInstr > 0);
8179	else
8180	cbInstr = 0;
8181	}
8182
8183	/** @todo The guest has likely hit a contended spinlock. We might want to
8184	* poke a schedule different guest VCPU. */
8185	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8186	return VINF_EM_RAW_INTERRUPT;
8187	}
8188
8189
8190	/**
8191	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8192	* \#VMEXIT.
8193	*/
8194	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8195	{
8196	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8197	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8198	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8199
8200	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8201	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8202	}
8203
8204
8205	/**
8206	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8207	*/
8208	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8209	{
8210	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8211
8212	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8213	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8214	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8215
8216	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8217	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8218	* handler executes. Yes, x86 is ugly.
8219	*/
8220	return VINF_EM_RAW_EMULATE_INSTR;
8221	}
8222
8223
8224	/**
8225	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8226	* Conditional \#VMEXIT.
8227	*/
8228	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8229	{
8230	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8231	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8232	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8233
8234	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8235	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8236	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8237	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8238	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8239	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8240
8241	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
8242	if (pVM->hmr0.s.fNestedPaging)
8243	{
8244	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8245	if ( !pSvmTransient->fVectoringDoublePF
8246	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8247	{
8248	/* A genuine guest #PF, reflect it to the guest. */
8249	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8250	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8251	uFaultAddress, uErrCode));
8252	}
8253	else
8254	{
8255	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8256	hmR0SvmSetPendingXcptDF(pVCpu);
8257	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8258	}
8259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8260	return VINF_SUCCESS;
8261	}
8262	#endif
8263
8264	Assert(!pVM->hmr0.s.fNestedPaging);
8265
8266	/*
8267	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8268	*/
8269	if ( pVM->hm.s.fTprPatchingAllowed
8270	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8271	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8272	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8273	&& !CPUMIsGuestInLongModeEx(pCtx)
8274	&& !CPUMGetGuestCPL(pVCpu)
8275	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8276	{
8277	RTGCPHYS GCPhysApicBase;
8278	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
8279	GCPhysApicBase &= PAGE_BASE_GC_MASK;
8280
8281	/* Check if the page at the fault-address is the APIC base. */
8282	PGMPTWALK Walk;
8283	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8284	if ( rc2 == VINF_SUCCESS
8285	&& Walk.GCPhys == GCPhysApicBase)
8286	{
8287	/* Only attempt to patch the instruction once. */
8288	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8289	if (!pPatch)
8290	return VINF_EM_HM_PATCH_TPR_INSTR;
8291	}
8292	}
8293
8294	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8295	pCtx->rip, uErrCode, pCtx->cr3));
8296
8297	/*
8298	* If it's a vectoring #PF, emulate injecting the original event injection as
8299	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8300	* event injection that caused a #PF. See @bugref{6607}.
8301	*/
8302	if (pSvmTransient->fVectoringPF)
8303	{
8304	Assert(pVCpu->hm.s.Event.fPending);
8305	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8306	}
8307
8308	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8309	int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
8310
8311	Log4Func(("#PF: rc=%Rrc\n", rc));
8312
8313	if (rc == VINF_SUCCESS)
8314	{
8315	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8316	TRPMResetTrap(pVCpu);
8317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8318	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8319	return rc;
8320	}
8321
8322	if (rc == VINF_EM_RAW_GUEST_TRAP)
8323	{
8324	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8325
8326	/*
8327	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8328	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8329	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8330	*/
8331	if ( !pSvmTransient->fVectoringDoublePF
8332	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8333	{
8334	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
8335	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8336	TRPMResetTrap(pVCpu);
8337
8338	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8339	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8340	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8341	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8342	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8343	#endif
8344
8345	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8346	}
8347	else
8348	{
8349	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8350	TRPMResetTrap(pVCpu);
8351	hmR0SvmSetPendingXcptDF(pVCpu);
8352	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8353	}
8354
8355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8356	return VINF_SUCCESS;
8357	}
8358
8359	TRPMResetTrap(pVCpu);
8360	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8361	return rc;
8362	}
8363
8364
8365	/**
8366	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8367	* Conditional \#VMEXIT.
8368	*/
8369	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8370	{
8371	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8372	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8373	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8374
8375	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8376	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8377	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8378
8379	/** @todo if we accumulate more optional stuff here, we ought to combine the
8380	* reading of opcode bytes to avoid doing more than once. */
8381
8382	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8383	if (pVCpu->hm.s.fGIMTrapXcptUD)
8384	{
8385	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8386	uint8_t cbInstr = 0;
8387	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8388	if (rcStrict == VINF_SUCCESS)
8389	{
8390	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8391	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8392	rcStrict = VINF_SUCCESS;
8393	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8394	}
8395	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8396	rcStrict = VINF_SUCCESS;
8397	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8398	rcStrict = VINF_GIM_R3_HYPERCALL;
8399	else
8400	{
8401	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8402	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8403	}
8404	}
8405
8406	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8407	{
8408	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
8409	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8410	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8411	{
8412	/* Ideally, IEM should just handle all these special #UD situations, but
8413	we don't quite trust things to behave optimially when doing that. So,
8414	for now we'll restrict ourselves to a handful of possible sysenter and
8415	sysexit encodings that we filter right here. */
8416	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8417	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8418	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8419	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8420	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8421	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8422	{
8423	cbInstr = cbMin;
8424	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8425	AssertRCStmt(rc2, cbInstr = 0);
8426	}
8427	else
8428	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8429	if ( cbInstr == 0 /* read error */
8430	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8431	\|\| ( uCpl == 0
8432	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8433	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8434	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
8435	{
8436	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8437	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8438	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8439	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), GCPtrInstr, abInstr, cbInstr);
8440	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8441	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8442	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8443	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8444	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8445	if (rcStrict == VINF_IEM_RAISED_XCPT)
8446	rcStrict = VINF_SUCCESS;
8447	return rcStrict;
8448	}
8449	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8450	}
8451	else
8452	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8453	}
8454
8455	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8456	if (RT_FAILURE(rcStrict))
8457	{
8458	hmR0SvmSetPendingXcptUD(pVCpu);
8459	rcStrict = VINF_SUCCESS;
8460	}
8461
8462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8463	return rcStrict;
8464	}
8465
8466
8467	/**
8468	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8469	* Conditional \#VMEXIT.
8470	*/
8471	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8472	{
8473	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8474	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8476
8477	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8478	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8479
8480	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8481	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8482
8483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8484
8485	if (!(pCtx->cr0 & X86_CR0_NE))
8486	{
8487	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8488	PDISSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8489	unsigned cbInstr;
8490	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbInstr);
8491	if (RT_SUCCESS(rc))
8492	{
8493	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8494	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8495	if (RT_SUCCESS(rc))
8496	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8497	}
8498	else
8499	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8500	return rc;
8501	}
8502
8503	hmR0SvmSetPendingXcptMF(pVCpu);
8504	return VINF_SUCCESS;
8505	}
8506
8507
8508	/**
8509	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8510	* \#VMEXIT.
8511	*/
8512	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8513	{
8514	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8515	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8516	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8518
8519	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8520	{
8521	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8522	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8523	}
8524
8525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8526
8527	/*
8528	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8529	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8530	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8531	*/
8532	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8533	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8534	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8535	int rc = DBGFTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8536	if (rc == VINF_EM_RAW_GUEST_TRAP)
8537	{
8538	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8539	if (CPUMIsHyperDebugStateActive(pVCpu))
8540	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
8541
8542	/* Reflect the exception back to the guest. */
8543	hmR0SvmSetPendingXcptDB(pVCpu);
8544	rc = VINF_SUCCESS;
8545	}
8546
8547	/*
8548	* Update DR6.
8549	*/
8550	if (CPUMIsHyperDebugStateActive(pVCpu))
8551	{
8552	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8553	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8554	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8555	}
8556	else
8557	{
8558	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8559	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8560	}
8561
8562	return rc;
8563	}
8564
8565
8566	/**
8567	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8568	* Conditional \#VMEXIT.
8569	*/
8570	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8571	{
8572	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8573	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8574	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8575
8576	SVMEVENT Event;
8577	Event.u = 0;
8578	Event.n.u1Valid = 1;
8579	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8580	Event.n.u8Vector = X86_XCPT_AC;
8581	Event.n.u1ErrorCodeValid = 1;
8582	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8583	return VINF_SUCCESS;
8584	}
8585
8586
8587	/**
8588	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8589	* Conditional \#VMEXIT.
8590	*/
8591	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8592	{
8593	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8594	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8595	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8597
8598	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8599	VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
8600	if (rc == VINF_EM_RAW_GUEST_TRAP)
8601	{
8602	SVMEVENT Event;
8603	Event.u = 0;
8604	Event.n.u1Valid = 1;
8605	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8606	Event.n.u8Vector = X86_XCPT_BP;
8607	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8608	rc = VINF_SUCCESS;
8609	}
8610
8611	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
8612	return rc;
8613	}
8614
8615
8616	/**
8617	* Hacks its way around the lovely mesa driver's backdoor accesses.
8618	*
8619	* @sa hmR0VmxHandleMesaDrvGp
8620	*/
8621	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8622	{
8623	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
8624	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8625	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8626	RT_NOREF(pCtx, pVmcb);
8627
8628	/* For now we'll just skip the instruction. */
8629	hmR0SvmAdvanceRip(pVCpu, 1);
8630	return VINF_SUCCESS;
8631	}
8632
8633
8634	/**
8635	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8636	* backdoor logging w/o checking what it is running inside.
8637	*
8638	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8639	* backdoor port and magic numbers loaded in registers.
8640	*
8641	* @returns true if it is, false if it isn't.
8642	* @sa hmR0VmxIsMesaDrvGp
8643	*/
8644	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8645	{
8646	/* Check magic and port. */
8647	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
8648	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
8649	if (pCtx->dx != UINT32_C(0x5658))
8650	return false;
8651	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8652	return false;
8653
8654	/* Check that it is #GP(0). */
8655	if (pVmcb->ctrl.u64ExitInfo1 != 0)
8656	return false;
8657
8658	/* Flat ring-3 CS. */
8659	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
8660	if (pVmcb->guest.u8CPL != 3)
8661	return false;
8662	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8663	return false;
8664
8665	/* 0xed: IN eAX,dx */
8666	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8667	{
8668	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
8669	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8670	uint8_t abInstr[1];
8671	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8672	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
8673	if (RT_FAILURE(rc))
8674	return false;
8675	if (abInstr[0] != 0xed)
8676	return false;
8677	}
8678	else
8679	{
8680	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
8681	if (pVmcb->ctrl.abInstr[0] != 0xed)
8682	return false;
8683	}
8684	return true;
8685	}
8686
8687
8688	/**
8689	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8690	* Conditional \#VMEXIT.
8691	*/
8692	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8693	{
8694	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8695	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8696	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8697
8698	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8699	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8700
8701	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8702	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8703	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8704	{
8705	SVMEVENT Event;
8706	Event.u = 0;
8707	Event.n.u1Valid = 1;
8708	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8709	Event.n.u8Vector = X86_XCPT_GP;
8710	Event.n.u1ErrorCodeValid = 1;
8711	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8712	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8713	return VINF_SUCCESS;
8714	}
8715	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8716	}
8717
8718
8719	/**
8720	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8721	*/
8722	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8723	{
8724	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8725	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8726
8727	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8728	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8729	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8730	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8731	Assert(uVector <= X86_XCPT_LAST);
8732	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8733
8734	SVMEVENT Event;
8735	Event.u = 0;
8736	Event.n.u1Valid = 1;
8737	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8738	Event.n.u8Vector = uVector;
8739	switch (uVector)
8740	{
8741	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8742	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8743	case X86_XCPT_DF:
8744	case X86_XCPT_TS:
8745	case X86_XCPT_NP:
8746	case X86_XCPT_SS:
8747	case X86_XCPT_GP:
8748	case X86_XCPT_AC:
8749	{
8750	Event.n.u1ErrorCodeValid = 1;
8751	Event.n.u32ErrorCode = uErrCode;
8752	break;
8753	}
8754	}
8755
8756	#ifdef VBOX_WITH_STATISTICS
8757	switch (uVector)
8758	{
8759	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8760	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8761	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8762	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8763	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8764	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8765	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8766	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8767	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8768	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8769	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8770	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8771	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8772	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8773	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8774	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8775	default:
8776	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8777	break;
8778	}
8779	#endif
8780
8781	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8782	return VINF_SUCCESS;
8783	}
8784
8785
8786	/**
8787	* \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8788	*/
8789	HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8790	{
8791	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8792	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8793
8794	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8795	SVMEVENT Event;
8796	Event.u = 0;
8797	Event.n.u1Valid = 1;
8798	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8799	Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8800	Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8801	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8802	return VINF_SUCCESS;
8803	}
8804
8805
8806	/**
8807	* Generic exit handler that interprets the current instruction
8808	*
8809	* Useful exit that only gets triggered by dtrace and the debugger. Caller does
8810	* the exit logging, and this function does the rest.
8811	*/
8812	static VBOXSTRICTRC hmR0SvmExitInterpretInstruction(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient,
8813	uint64_t fExtraImport, uint64_t fHmChanged)
8814	{
8815	#if 1
8816	RT_NOREF(pSvmTransient);
8817	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fExtraImport);
8818	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
8819	if (rcStrict == VINF_SUCCESS)
8820	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RIP);
8821	else
8822	{
8823	Log4Func(("IEMExecOne -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
8824	if (rcStrict == VINF_IEM_RAISED_XCPT)
8825	{
8826	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK \| fHmChanged);
8827	rcStrict = VINF_SUCCESS;
8828	}
8829	else
8830	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged);
8831	}
8832	return rcStrict;
8833	#else
8834	RT_NOREF(pVCpu, pSvmTransient, fExtraImport, fHmChanged);
8835	return VINF_EM_RAW_EMULATE_INSTR;
8836	#endif
8837	}
8838
8839
8840	/**
8841	* \#VMEXIT handler for STR. Conditional \#VMEXIT (debug).
8842	*/
8843	HMSVM_EXIT_DECL hmR0SvmExitTrRead(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8844	{
8845	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8846	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8847	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR, 0);
8848	}
8849
8850
8851	/**
8852	* \#VMEXIT handler for LTR. Conditional \#VMEXIT (OS/2 TLB workaround, debug).
8853	*/
8854	HMSVM_EXIT_DECL hmR0SvmExitTrWrite(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8855	{
8856	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8857
8858	/* Workaround for lack of TLB flushing in OS/2 when returning to protected
8859	mode after a real mode call (like a BIOS call). See ticketref:20625
8860	comment 14. */
8861	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8862	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
8863	{
8864	Log4Func(("%04x:%08RX64 TLB flush\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8865	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
8866	}
8867	else
8868	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8869
8870	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR, HM_CHANGED_GUEST_TR);
8871	}
8872
8873
8874	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8875	/**
8876	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8877	*/
8878	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8879	{
8880	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8881
8882	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8883	Assert(pVmcb);
8884	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8885
8886	VBOXSTRICTRC rcStrict;
8887	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8888	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8889	if (fSupportsNextRipSave)
8890	{
8891	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8892	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8893	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8894	}
8895	else
8896	{
8897	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8898	rcStrict = IEMExecOne(pVCpu);
8899	}
8900
8901	if (rcStrict == VINF_SUCCESS)
8902	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8903	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8904	{
8905	rcStrict = VINF_SUCCESS;
8906	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8907	}
8908	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8909	return rcStrict;
8910	}
8911
8912
8913	/**
8914	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8915	*/
8916	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8917	{
8918	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8919
8920	/*
8921	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8922	* we only intercept STGI when events are pending for GIF to become 1.
8923	*/
8924	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8925	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8926	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8927
8928	VBOXSTRICTRC rcStrict;
8929	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8930	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8931	if (fSupportsNextRipSave)
8932	{
8933	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8934	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8935	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8936	}
8937	else
8938	{
8939	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8940	rcStrict = IEMExecOne(pVCpu);
8941	}
8942
8943	if (rcStrict == VINF_SUCCESS)
8944	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8945	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8946	{
8947	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8948	rcStrict = VINF_SUCCESS;
8949	}
8950	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8951	return rcStrict;
8952	}
8953
8954
8955	/**
8956	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
8957	*/
8958	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8959	{
8960	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8961
8962	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8963	Assert(pVmcb);
8964	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
8965
8966	VBOXSTRICTRC rcStrict;
8967	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8968	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
8969	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
8970	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
8971	if (fSupportsNextRipSave)
8972	{
8973	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8974	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8975	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
8976	}
8977	else
8978	{
8979	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8980	rcStrict = IEMExecOne(pVCpu);
8981	}
8982
8983	if (rcStrict == VINF_SUCCESS)
8984	{
8985	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
8986	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
8987	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
8988	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
8989	}
8990	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8991	{
8992	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8993	rcStrict = VINF_SUCCESS;
8994	}
8995	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8996	return rcStrict;
8997	}
8998
8999
9000	/**
9001	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
9002	*/
9003	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9004	{
9005	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9006
9007	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9008	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9009
9010	VBOXSTRICTRC rcStrict;
9011	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9012	if (fSupportsNextRipSave)
9013	{
9014	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9015	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9016	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
9017	}
9018	else
9019	{
9020	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9021	rcStrict = IEMExecOne(pVCpu);
9022	}
9023
9024	if (rcStrict == VINF_IEM_RAISED_XCPT)
9025	{
9026	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9027	rcStrict = VINF_SUCCESS;
9028	}
9029	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9030	return rcStrict;
9031	}
9032
9033
9034	/**
9035	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
9036	*/
9037	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9038	{
9039	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9040
9041	VBOXSTRICTRC rcStrict;
9042	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9043	if (fSupportsNextRipSave)
9044	{
9045	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9046	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9047	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9048	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
9049	}
9050	else
9051	{
9052	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9053	rcStrict = IEMExecOne(pVCpu);
9054	}
9055
9056	if (rcStrict == VINF_IEM_RAISED_XCPT)
9057	{
9058	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9059	rcStrict = VINF_SUCCESS;
9060	}
9061	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9062	return rcStrict;
9063	}
9064
9065
9066	/**
9067	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
9068	*/
9069	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9070	{
9071	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9072	/* We shall import the entire state here, just in case we enter and continue execution of
9073	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
9074	could lose lose part of CPU state. */
9075	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
9076
9077	VBOXSTRICTRC rcStrict;
9078	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9079	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
9080	if (fSupportsNextRipSave)
9081	{
9082	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9083	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9084	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
9085	}
9086	else
9087	{
9088	/* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
9089	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
9090	instruction itself, see @bugref{7243#c126} */
9091	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), NULL /* pcbWritten */);
9092	}
9093	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
9094
9095	if (rcStrict == VINF_SUCCESS)
9096	{
9097	rcStrict = VINF_SVM_VMRUN;
9098	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9099	}
9100	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9101	{
9102	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9103	rcStrict = VINF_SUCCESS;
9104	}
9105	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9106	return rcStrict;
9107	}
9108
9109
9110	/**
9111	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9112	* Unconditional \#VMEXIT.
9113	*/
9114	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9115	{
9116	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9117	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9118
9119	if (pVCpu->hm.s.Event.fPending)
9120	{
9121	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9122	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9123	}
9124
9125	hmR0SvmSetPendingXcptDB(pVCpu);
9126	return VINF_SUCCESS;
9127	}
9128
9129
9130	/**
9131	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9132	* Conditional \#VMEXIT.
9133	*/
9134	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9135	{
9136	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9137	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9138
9139	SVMEVENT Event;
9140	Event.u = 0;
9141	Event.n.u1Valid = 1;
9142	Event.n.u3Type = SVM_EVENT_EXCEPTION;
9143	Event.n.u8Vector = X86_XCPT_BP;
9144	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9145	return VINF_SUCCESS;
9146	}
9147	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9148
9149	/** @} */
9150

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 93368

Download in other formats: