HMSVMR0.cpp@ 100357

Last change on this file since 100357 was 100357, checked in by vboxsync, 17 months ago
Runtime/RTR0MemObj*: Add PhysHighest parameter to RTR0MemObjAllocCont to indicate the maximum allowed physical address for an allocation, bugref:10457 [second attempt]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 380.3 KB

Line
1	/* $Id: HMSVMR0.cpp 100357 2023-07-04 07:00:26Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_HM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/thread.h>
36
37	#include <VBox/vmm/pdmapi.h>
38	#include <VBox/vmm/dbgf.h>
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/iom.h>
41	#include <VBox/vmm/tm.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/gcm.h>
44	#include <VBox/vmm/gim.h>
45	#include <VBox/vmm/apic.h>
46	#include "HMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include <VBox/err.h>
49	#include "HMSVMR0.h"
50	#include "dtrace/VBoxVMM.h"
51
52	#ifdef DEBUG_ramshankar
53	# define HMSVM_SYNC_FULL_GUEST_STATE
54	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
55	# define HMSVM_ALWAYS_TRAP_PF
56	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	#ifdef VBOX_WITH_STATISTICS
64	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
66	if ((u64ExitCode) == SVM_EXIT_NPF) \
67	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
68	else \
69	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
70	} while (0)
71
72	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
74	if ((u64ExitCode) == SVM_EXIT_NPF) \
75	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
76	else \
77	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
78	} while (0)
79
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
81	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
82	if ((u64ExitCode) == SVM_EXIT_NPF) \
83	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
84	else \
85	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
86	} while (0)
87	#else
88	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
89	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
90	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
91	#endif /* !VBOX_WITH_STATISTICS */
92
93	/** If we decide to use a function table approach this can be useful to
94	* switch to a "static DECLCALLBACK(int)". */
95	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
96
97	/**
98	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
99	* guest using hardware-assisted SVM.
100	*
101	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
102	* are swapped and restored across the world-switch and also registers like
103	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
104	* \#VMEXIT.
105	*/
106	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
107	\| CPUMCTX_EXTRN_RFLAGS \
108	\| CPUMCTX_EXTRN_RAX \
109	\| CPUMCTX_EXTRN_RSP \
110	\| CPUMCTX_EXTRN_SREG_MASK \
111	\| CPUMCTX_EXTRN_CR0 \
112	\| CPUMCTX_EXTRN_CR2 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_TABLE_MASK \
115	\| CPUMCTX_EXTRN_DR6 \
116	\| CPUMCTX_EXTRN_DR7 \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_HWVIRT \
121	\| CPUMCTX_EXTRN_INHIBIT_INT \
122	\| CPUMCTX_EXTRN_HM_SVM_MASK)
123
124	/**
125	* Subset of the guest-CPU state that is shared between the guest and host.
126	*/
127	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
128
129	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
130	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
131	do { \
132	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
133	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
134	} while (0)
135
136	/** Assert that the required state bits are fetched. */
137	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
151
152	/** Assert that we're not executing a nested-guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
155	#else
156	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
157	#endif
158
159	/** Assert that we're executing a nested-guest. */
160	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
161	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
162	#else
163	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
164	#endif
165
166	/** Macro for checking and returning from the using function for
167	* \#VMEXIT intercepts that maybe caused during delivering of another
168	* event in the guest. */
169	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else if ( rc == VINF_EM_RESET \
177	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
178	{ \
179	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
180	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
181	} \
182	else \
183	return rc; \
184	} while (0)
185	#else
186	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
187	do \
188	{ \
189	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
190	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
191	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
192	else \
193	return rc; \
194	} while (0)
195	#endif
196
197	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
198	* instruction that exited. */
199	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
200	do { \
201	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
202	(a_rc) = VINF_EM_DBG_STEPPED; \
203	} while (0)
204
205	/** Validate segment descriptor granularity bit. */
206	#ifdef VBOX_STRICT
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
208	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
209	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
210	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
211	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
212	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
213	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
214	#else
215	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
216	#endif
217
218	/**
219	* Exception bitmap mask for all contributory exceptions.
220	*
221	* Page fault is deliberately excluded here as it's conditional as to whether
222	* it's contributory or benign. Page faults are handled separately.
223	*/
224	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
225	\| RT_BIT(X86_XCPT_DE))
226
227	/**
228	* Mandatory/unconditional guest control intercepts.
229	*
230	* SMIs can and do happen in normal operation. We need not intercept them
231	* while executing the guest (or nested-guest).
232	*/
233	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
234	\| SVM_CTRL_INTERCEPT_NMI \
235	\| SVM_CTRL_INTERCEPT_INIT \
236	\| SVM_CTRL_INTERCEPT_RDPMC \
237	\| SVM_CTRL_INTERCEPT_CPUID \
238	\| SVM_CTRL_INTERCEPT_RSM \
239	\| SVM_CTRL_INTERCEPT_HLT \
240	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
241	\| SVM_CTRL_INTERCEPT_MSR_PROT \
242	\| SVM_CTRL_INTERCEPT_INVLPGA \
243	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
244	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
245	\| SVM_CTRL_INTERCEPT_VMRUN \
246	\| SVM_CTRL_INTERCEPT_SKINIT \
247	\| SVM_CTRL_INTERCEPT_WBINVD \
248	\| SVM_CTRL_INTERCEPT_MONITOR \
249	\| SVM_CTRL_INTERCEPT_MWAIT \
250	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
251	\| SVM_CTRL_INTERCEPT_XSETBV)
252
253	/** @name VMCB Clean Bits.
254	*
255	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
256	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
257	* memory.
258	*
259	* @{ */
260	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
261	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
262	/** I/O permission bitmap, MSR permission bitmap. */
263	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
264	/** ASID. */
265	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
266	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
267	V_INTR_VECTOR. */
268	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
269	/** Nested Paging: Nested CR3 (nCR3), PAT. */
270	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
271	/** Control registers (CR0, CR3, CR4, EFER). */
272	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
273	/** Debug registers (DR6, DR7). */
274	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
275	/** GDT, IDT limit and base. */
276	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
277	/** Segment register: CS, SS, DS, ES limit and base. */
278	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
279	/** CR2.*/
280	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
281	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
282	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
283	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
284	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
285	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
286	/** Mask of all valid VMCB Clean bits. */
287	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
288	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
289	\| HMSVM_VMCB_CLEAN_ASID \
290	\| HMSVM_VMCB_CLEAN_INT_CTRL \
291	\| HMSVM_VMCB_CLEAN_NP \
292	\| HMSVM_VMCB_CLEAN_CRX_EFER \
293	\| HMSVM_VMCB_CLEAN_DRX \
294	\| HMSVM_VMCB_CLEAN_DT \
295	\| HMSVM_VMCB_CLEAN_SEG \
296	\| HMSVM_VMCB_CLEAN_CR2 \
297	\| HMSVM_VMCB_CLEAN_LBR \
298	\| HMSVM_VMCB_CLEAN_AVIC)
299	/** @} */
300
301	/**
302	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
303	*/
304	typedef enum SVMMSREXITREAD
305	{
306	/** Reading this MSR causes a \#VMEXIT. */
307	SVMMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a \#VMEXIT. */
309	SVMMSREXIT_PASSTHRU_READ
310	} SVMMSREXITREAD;
311
312	/**
313	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
314	*/
315	typedef enum SVMMSREXITWRITE
316	{
317	/** Writing to this MSR causes a \#VMEXIT. */
318	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a \#VMEXIT. */
320	SVMMSREXIT_PASSTHRU_WRITE
321	} SVMMSREXITWRITE;
322
323	/**
324	* SVM \#VMEXIT handler.
325	*
326	* @returns Strict VBox status code.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pSvmTransient Pointer to the SVM-transient structure.
329	*/
330	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
331
332
333	/*********************************************************************************************************************************
334	* Internal Functions *
335	*********************************************************************************************************************************/
336	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
337	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
338
339
340	/** @name \#VMEXIT handlers.
341	* @{
342	*/
343	static FNSVMEXITHANDLER hmR0SvmExitIntr;
344	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
345	static FNSVMEXITHANDLER hmR0SvmExitInvd;
346	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
347	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
348	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
349	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
350	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
351	static FNSVMEXITHANDLER hmR0SvmExitHlt;
352	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
353	static FNSVMEXITHANDLER hmR0SvmExitMwait;
354	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
355	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
356	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
357	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
358	static FNSVMEXITHANDLER hmR0SvmExitMsr;
359	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
360	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
361	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
362	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
363	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
364	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
365	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
366	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
367	static FNSVMEXITHANDLER hmR0SvmExitPause;
368	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
369	static FNSVMEXITHANDLER hmR0SvmExitIret;
370	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
371	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
372	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
373	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
374	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
375	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
376	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
377	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
378	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
379	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
380	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
381	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
382	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
383	static FNSVMEXITHANDLER hmR0SvmExitClgi;
384	static FNSVMEXITHANDLER hmR0SvmExitStgi;
385	static FNSVMEXITHANDLER hmR0SvmExitVmload;
386	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
387	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
388	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
389	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
390	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
391	#endif
392	/** @} */
393
394	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
396	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
397	#endif
398	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	/** Ring-0 memory object for the IO bitmap. */
405	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
406	/** Physical address of the IO bitmap. */
407	static RTHCPHYS g_HCPhysIOBitmap;
408	/** Pointer to the IO bitmap. */
409	static R0PTRTYPE(void *) g_pvIOBitmap;
410
411	#ifdef VBOX_STRICT
412	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
413	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
414	# define HMSVM_LOG_CS RT_BIT_32(2)
415	# define HMSVM_LOG_SS RT_BIT_32(3)
416	# define HMSVM_LOG_FS RT_BIT_32(4)
417	# define HMSVM_LOG_GS RT_BIT_32(5)
418	# define HMSVM_LOG_LBR RT_BIT_32(6)
419	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
420	\| HMSVM_LOG_CR_REGS \
421	\| HMSVM_LOG_CS \
422	\| HMSVM_LOG_SS \
423	\| HMSVM_LOG_FS \
424	\| HMSVM_LOG_GS \
425	\| HMSVM_LOG_LBR)
426
427	/**
428	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
429	*
430	* @param pVCpu The cross context virtual CPU structure.
431	* @param pVmcb Pointer to the VM control block.
432	* @param pszPrefix Log prefix.
433	* @param fFlags Log flags, see HMSVM_LOG_XXX.
434	* @param uVerbose The verbosity level, currently unused.
435	*/
436	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
437	{
438	RT_NOREF2(pVCpu, uVerbose);
439	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
440
441	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
442	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
443
444	if (fFlags & HMSVM_LOG_RBP_RSP)
445	{
446	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
447	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
448	}
449
450	if (fFlags & HMSVM_LOG_CR_REGS)
451	{
452	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
453	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
454	}
455
456	if (fFlags & HMSVM_LOG_CS)
457	{
458	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
459	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
460	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
461	}
462	if (fFlags & HMSVM_LOG_SS)
463	{
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
465	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
466	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
467	}
468	if (fFlags & HMSVM_LOG_FS)
469	{
470	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
471	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
472	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
473	}
474	if (fFlags & HMSVM_LOG_GS)
475	{
476	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
477	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
478	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
479	}
480
481	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
482	if (fFlags & HMSVM_LOG_LBR)
483	{
484	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
485	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
486	}
487	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
488	}
489	#endif /* VBOX_STRICT */
490
491
492	/**
493	* Sets up and activates AMD-V on the current CPU.
494	*
495	* @returns VBox status code.
496	* @param pHostCpu The HM physical-CPU structure.
497	* @param pVM The cross context VM structure. Can be
498	* NULL after a resume!
499	* @param pvCpuPage Pointer to the global CPU page.
500	* @param HCPhysCpuPage Physical address of the global CPU page.
501	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
502	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
503	* unused).
504	*/
505	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
506	PCSUPHWVIRTMSRS pHwvirtMsrs)
507	{
508	Assert(!fEnabledByHost);
509	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
510	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
511	Assert(pvCpuPage); NOREF(pvCpuPage);
512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
513
514	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
515
516	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
517	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
518
519	/*
520	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
521	*/
522	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
523	if (u64HostEfer & MSR_K6_EFER_SVME)
524	{
525	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
526	if ( pVM
527	&& pVM->hm.s.svm.fIgnoreInUseError)
528	pHostCpu->fIgnoreAMDVInUseError = true;
529
530	if (!pHostCpu->fIgnoreAMDVInUseError)
531	{
532	ASMSetFlags(fEFlags);
533	return VERR_SVM_IN_USE;
534	}
535	}
536
537	/* Turn on AMD-V in the EFER MSR. */
538	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
539
540	/* Write the physical page address where the CPU will store the host state while executing the VM. */
541	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
542
543	/* Restore interrupts. */
544	ASMSetFlags(fEFlags);
545
546	/*
547	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
548	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
549	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
550	* entirely with before executing any guest code.
551	*/
552	pHostCpu->fFlushAsidBeforeUse = true;
553
554	/*
555	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
556	*/
557	++pHostCpu->cTlbFlushes;
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Deactivates AMD-V on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pHostCpu The HM physical-CPU structure.
568	* @param pvCpuPage Pointer to the global CPU page.
569	* @param HCPhysCpuPage Physical address of the global CPU page.
570	*/
571	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
572	{
573	RT_NOREF1(pHostCpu);
574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
575	AssertReturn( HCPhysCpuPage
576	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
580	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
581
582	/* Turn off AMD-V in the EFER MSR. */
583	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
584	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
585
586	/* Invalidate host state physical address. */
587	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
588
589	/* Restore interrupts. */
590	ASMSetFlags(fEFlags);
591
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Does global AMD-V initialization (called during module initialization).
598	*
599	* @returns VBox status code.
600	*/
601	VMMR0DECL(int) SVMR0GlobalInit(void)
602	{
603	/*
604	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
605	* intercept all IO accesses, it's done once globally here instead of per-VM.
606	*/
607	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
608	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT,
609	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
610	if (RT_FAILURE(rc))
611	return rc;
612
613	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
614	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
615
616	/* Set all bits to intercept all IO accesses. */
617	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
618
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Does global AMD-V termination (called during module termination).
625	*/
626	VMMR0DECL(void) SVMR0GlobalTerm(void)
627	{
628	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
629	{
630	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
631	g_pvIOBitmap = NULL;
632	g_HCPhysIOBitmap = 0;
633	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
634	}
635	}
636
637
638	/**
639	* Frees any allocated per-VCPU structures for a VM.
640	*
641	* @param pVM The cross context VM structure.
642	*/
643	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
644	{
645	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
646	{
647	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
648	AssertPtr(pVCpu);
649
650	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
651	{
652	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
653	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
654	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
655	}
656
657	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
658	{
659	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
660	pVCpu->hmr0.s.svm.pVmcb = NULL;
661	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
662	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
663	}
664
665	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
666	{
667	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
668	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
669	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
670	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
671	}
672	}
673	}
674
675
676	/**
677	* Sets pfnVMRun to the best suited variant.
678	*
679	* This must be called whenever anything changes relative to the SVMR0VMRun
680	* variant selection:
681	* - pVCpu->hm.s.fLoadSaveGuestXcr0
682	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
683	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
684	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
685	* - Perhaps: CPUMCTX.fXStateMask (windows only)
686	*
687	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
688	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
689	*/
690	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
691	{
692	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
693	{
694	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
695	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
696	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
697	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
698	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
699	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
700	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
701	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
702	};
703	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
704	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
705	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
706	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
707	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
708	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
709	}
710
711
712	/**
713	* Selector FNHMSVMVMRUN implementation.
714	*/
715	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
716	{
717	hmR0SvmUpdateVmRunFunction(pVCpu);
718	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
719	}
720
721
722	/**
723	* Does per-VM AMD-V initialization.
724	*
725	* @returns VBox status code.
726	* @param pVM The cross context VM structure.
727	*/
728	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
729	{
730	int rc = VERR_INTERNAL_ERROR_5;
731
732	/*
733	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
734	*/
735	uint32_t u32Family;
736	uint32_t u32Model;
737	uint32_t u32Stepping;
738	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
739	{
740	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
741	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
742	}
743
744	/*
745	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
746	*/
747	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
748	{
749	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
750	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
751	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
752	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
753	}
754
755	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
756	{
757	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
758
759	/*
760	* Initialize the hardware-assisted SVM guest-execution handler.
761	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
762	*/
763	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
764
765	/*
766	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
767	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
768	*/
769	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
770	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
771	if (RT_FAILURE(rc))
772	goto failure_cleanup;
773
774	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
775	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
776	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
777
778	/*
779	* Allocate one page for the guest-state VMCB.
780	*/
781	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
782	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
783	if (RT_FAILURE(rc))
784	goto failure_cleanup;
785
786	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
787	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
788	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
789
790	/*
791	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
792	* SVM to not require one.
793	*/
794	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
795	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
796	if (RT_FAILURE(rc))
797	goto failure_cleanup;
798
799	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
800	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
801	/* Set all bits to intercept all MSR accesses (changed later on). */
802	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
803	}
804
805	return VINF_SUCCESS;
806
807	failure_cleanup:
808	hmR0SvmFreeStructs(pVM);
809	return rc;
810	}
811
812
813	/**
814	* Does per-VM AMD-V termination.
815	*
816	* @returns VBox status code.
817	* @param pVM The cross context VM structure.
818	*/
819	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
820	{
821	hmR0SvmFreeStructs(pVM);
822	return VINF_SUCCESS;
823	}
824
825
826	/**
827	* Returns whether the VMCB Clean Bits feature is supported.
828	*
829	* @returns @c true if supported, @c false otherwise.
830	* @param pVCpu The cross context virtual CPU structure.
831	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
832	*/
833	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
834	{
835	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
836	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
837	if (!fIsNestedGuest)
838	return fHostVmcbCleanBits;
839	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
840	}
841
842
843	/**
844	* Returns whether the decode assists feature is supported.
845	*
846	* @returns @c true if supported, @c false otherwise.
847	* @param pVCpu The cross context virtual CPU structure.
848	*/
849	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
850	{
851	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
852	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
853	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
854	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
855	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
856	#endif
857	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
858	}
859
860
861	/**
862	* Returns whether the NRIP_SAVE feature is supported.
863	*
864	* @returns @c true if supported, @c false otherwise.
865	* @param pVCpu The cross context virtual CPU structure.
866	*/
867	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
868	{
869	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
870	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
871	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
872	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
873	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
874	#endif
875	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
876	}
877
878
879	/**
880	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
881	*
882	* @param pVCpu The cross context virtual CPU structure.
883	* @param pbMsrBitmap Pointer to the MSR bitmap.
884	* @param idMsr The MSR for which the permissions are being set.
885	* @param enmRead MSR read permissions.
886	* @param enmWrite MSR write permissions.
887	*
888	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
889	* caller needs to take care of this.
890	*/
891	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
892	SVMMSREXITWRITE enmWrite)
893	{
894	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
895	uint16_t offMsrpm;
896	uint8_t uMsrpmBit;
897	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
898	AssertRC(rc);
899
900	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
901	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
902
903	pbMsrBitmap += offMsrpm;
904	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
905	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
906	else
907	{
908	if (!fInNestedGuestMode)
909	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
910	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
911	else
912	{
913	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
914	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
915	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
916	else
917	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
918	}
919	#endif
920	}
921
922	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
923	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
924	else
925	{
926	if (!fInNestedGuestMode)
927	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
928	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
929	else
930	{
931	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
932	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
933	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
934	else
935	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
936	}
937	#endif
938	}
939	}
940
941
942	/**
943	* Sets up AMD-V for the specified VM.
944	* This function is only called once per-VM during initalization.
945	*
946	* @returns VBox status code.
947	* @param pVM The cross context VM structure.
948	*/
949	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
950	{
951	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
952	AssertReturn(pVM, VERR_INVALID_PARAMETER);
953
954	/*
955	* Validate and copy over some parameters.
956	*/
957	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
958	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
959	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
960	pVM->hmr0.s.fNestedPaging = fNestedPaging;
961	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
962
963	/*
964	* Determin some configuration parameters.
965	*/
966	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
967	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
968	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
969
970	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
971	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
972
973	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
974	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
975	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
976
977	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
978	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
979	#endif
980
981	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
982	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
983	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
984	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
985
986	/* Always trap #AC for reasons of security. */
987	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
988
989	/* Always trap #DB for reasons of security. */
990	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
991
992	/* Trap exceptions unconditionally (debug purposes). */
993	#ifdef HMSVM_ALWAYS_TRAP_PF
994	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
995	#endif
996	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
997	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
998	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
999	\| RT_BIT_32(X86_XCPT_DE)
1000	\| RT_BIT_32(X86_XCPT_NM)
1001	\| RT_BIT_32(X86_XCPT_UD)
1002	\| RT_BIT_32(X86_XCPT_NP)
1003	\| RT_BIT_32(X86_XCPT_SS)
1004	\| RT_BIT_32(X86_XCPT_GP)
1005	\| RT_BIT_32(X86_XCPT_PF)
1006	\| RT_BIT_32(X86_XCPT_MF)
1007	;
1008	#endif
1009
1010	/* Apply the exceptions intercepts needed by the GIM provider. */
1011	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1012	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1013
1014	/* Apply the exceptions intercepts needed by the GCM fixers. */
1015	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1016	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1017
1018	/* The mesa 3d driver hack needs #GP. */
1019	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1020	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1021
1022	/* Set up unconditional intercepts and conditions. */
1023	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1024	\| SVM_CTRL_INTERCEPT_VMMCALL
1025	\| SVM_CTRL_INTERCEPT_VMSAVE
1026	\| SVM_CTRL_INTERCEPT_VMLOAD
1027	\| SVM_CTRL_INTERCEPT_CLGI
1028	\| SVM_CTRL_INTERCEPT_STGI;
1029
1030	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1031	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1032	#endif
1033
1034	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1035	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1036	{
1037	/* Virtualized VMSAVE/VMLOAD. */
1038	if (fUseVirtVmsaveVmload)
1039	{
1040	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1041	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1042	\| SVM_CTRL_INTERCEPT_VMLOAD);
1043	}
1044	else
1045	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1046
1047	/* Virtual GIF. */
1048	if (fUseVGif)
1049	{
1050	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1051	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1052	\| SVM_CTRL_INTERCEPT_STGI);
1053	}
1054	else
1055	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1056	}
1057	else
1058	#endif
1059	{
1060	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1061	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1062	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1063	}
1064
1065	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1066	AVX (for XCR0 syncing during worlds switching). */
1067	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1068
1069	/* Intercept all DRx reads and writes by default. Changed later on. */
1070	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1071	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1072
1073	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1074	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1075
1076	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1077	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1078	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1079
1080	/* Set the IO permission bitmap physical addresses. */
1081	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1082
1083	/* LBR virtualization. */
1084	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1085
1086	/* The host ASID MBZ, for the guest start with 1. */
1087	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1088
1089	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1090	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1091
1092	/* Without Nested Paging, we need additionally intercepts. */
1093	if (!fNestedPaging)
1094	{
1095	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1096	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1097	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1098
1099	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1100	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1101	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1102
1103	/* Page faults must be intercepted to implement shadow paging. */
1104	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1105	}
1106
1107	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1108	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1109	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1110
1111	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1112	if (fUsePauseFilter)
1113	{
1114	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1115	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1116	if (fPauseFilterThreshold)
1117	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1118	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1119	}
1120
1121	/*
1122	* Setup the MSR permission bitmap.
1123	* The following MSRs are saved/restored automatically during the world-switch.
1124	* Don't intercept guest read/write accesses to these MSRs.
1125	*/
1126	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1127	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1128	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1129	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1130	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1131	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1132	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1133	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1134	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1135	{
1136	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1137	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1138	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1139	}
1140	else
1141	{
1142	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1143	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1144	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1145	}
1146	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1147
1148	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1149	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1150
1151	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1152	{
1153	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1154	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1155	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1156	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1157
1158	/* Copy the VMCB control area. */
1159	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1160
1161	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1162	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1163	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1164	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1165
1166	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1167	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1168
1169	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1170	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1171	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1172	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1173	}
1174
1175	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1176	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1177	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1178	#else
1179	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1180	#endif
1181	return VINF_SUCCESS;
1182	}
1183
1184
1185	/**
1186	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1187	*
1188	* @returns Pointer to the current context VMCB.
1189	* @param pVCpu The cross context virtual CPU structure.
1190	*/
1191	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1192	{
1193	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1194	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1195	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1196	#endif
1197	return pVCpu->hmr0.s.svm.pVmcb;
1198	}
1199
1200
1201	/**
1202	* Gets a pointer to the nested-guest VMCB cache.
1203	*
1204	* @returns Pointer to the nested-guest VMCB cache.
1205	* @param pVCpu The cross context virtual CPU structure.
1206	*/
1207	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1208	{
1209	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1210	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1211	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1212	#else
1213	RT_NOREF(pVCpu);
1214	return NULL;
1215	#endif
1216	}
1217
1218
1219	/**
1220	* Invalidates a guest page by guest virtual address.
1221	*
1222	* @returns VBox status code.
1223	* @param pVCpu The cross context virtual CPU structure.
1224	* @param GCVirt Guest virtual address of the page to invalidate.
1225	*/
1226	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1227	{
1228	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1229
1230	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1231
1232	/* Skip it if a TLB flush is already pending. */
1233	if (!fFlushPending)
1234	{
1235	Log4Func(("%#RGv\n", GCVirt));
1236
1237	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1238	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1239
1240	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1242	}
1243	return VINF_SUCCESS;
1244	}
1245
1246
1247	/**
1248	* Flushes the appropriate tagged-TLB entries.
1249	*
1250	* @param pHostCpu The HM physical-CPU structure.
1251	* @param pVCpu The cross context virtual CPU structure.
1252	* @param pVmcb Pointer to the VM control block.
1253	*/
1254	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1255	{
1256	/*
1257	* Force a TLB flush for the first world switch if the current CPU differs from the one
1258	* we ran on last. This can happen both for start & resume due to long jumps back to
1259	* ring-3.
1260	*
1261	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1262	* correlation between it and the physical CPU.
1263	*
1264	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1265	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1266	*/
1267	bool fNewAsid = false;
1268	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1269	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1270	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1271	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1272	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1273	#endif
1274	)
1275	{
1276	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1277	pVCpu->hmr0.s.fForceTLBFlush = true;
1278	fNewAsid = true;
1279	}
1280
1281	/* Set TLB flush state as checked until we return from the world switch. */
1282	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1283
1284	/* Check for explicit TLB flushes. */
1285	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1286	{
1287	pVCpu->hmr0.s.fForceTLBFlush = true;
1288	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1289	}
1290
1291	/*
1292	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1293	* This Host CPU requirement takes precedence.
1294	*/
1295	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1296	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1297	{
1298	pHostCpu->uCurrentAsid = 1;
1299	pVCpu->hmr0.s.uCurrentAsid = 1;
1300	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1301	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1302	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1303
1304	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1305	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1306	}
1307	else
1308	{
1309	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1310	if (pVCpu->hmr0.s.fForceTLBFlush)
1311	{
1312	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1313	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1314
1315	if (fNewAsid)
1316	{
1317	++pHostCpu->uCurrentAsid;
1318
1319	bool fHitASIDLimit = false;
1320	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1321	{
1322	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1323	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1324	fHitASIDLimit = true;
1325	}
1326
1327	if ( fHitASIDLimit
1328	\|\| pHostCpu->fFlushAsidBeforeUse)
1329	{
1330	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1331	pHostCpu->fFlushAsidBeforeUse = false;
1332	}
1333
1334	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1335	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1336	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1337	}
1338	else
1339	{
1340	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1341	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1342	else
1343	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1344	}
1345
1346	pVCpu->hmr0.s.fForceTLBFlush = false;
1347	}
1348	}
1349
1350	/* Update VMCB with the ASID. */
1351	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1352	{
1353	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1354	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1355	}
1356
1357	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1358	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1359	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1360	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1361	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1362	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1363	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1364	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1365
1366	#ifdef VBOX_WITH_STATISTICS
1367	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1368	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1369	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1370	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1371	{
1372	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1373	}
1374	else
1375	{
1376	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1377	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1378	}
1379	#endif
1380	}
1381
1382
1383	/**
1384	* Sets an exception intercept in the specified VMCB.
1385	*
1386	* @param pVmcb Pointer to the VM control block.
1387	* @param uXcpt The exception (X86_XCPT_*).
1388	*/
1389	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1390	{
1391	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1392	{
1393	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1394	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1395	}
1396	}
1397
1398
1399	/**
1400	* Clears an exception intercept in the specified VMCB.
1401	*
1402	* @param pVCpu The cross context virtual CPU structure.
1403	* @param pVmcb Pointer to the VM control block.
1404	* @param uXcpt The exception (X86_XCPT_*).
1405	*
1406	* @remarks This takes into account if we're executing a nested-guest and only
1407	* removes the exception intercept if both the guest -and- nested-guest
1408	* are not intercepting it.
1409	*/
1410	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1411	{
1412	Assert(uXcpt != X86_XCPT_DB);
1413	Assert(uXcpt != X86_XCPT_AC);
1414	Assert(uXcpt != X86_XCPT_GP);
1415	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1416	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1417	{
1418	bool fRemove = true;
1419	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1420	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1421	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1422	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1423	{
1424	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1425	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1426	}
1427	# else
1428	RT_NOREF(pVCpu);
1429	# endif
1430	if (fRemove)
1431	{
1432	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1433	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1434	}
1435	}
1436	#else
1437	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1438	#endif
1439	}
1440
1441
1442	/**
1443	* Sets a control intercept in the specified VMCB.
1444	*
1445	* @param pVmcb Pointer to the VM control block.
1446	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1447	*/
1448	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1449	{
1450	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1451	{
1452	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1453	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1454	}
1455	}
1456
1457
1458	/**
1459	* Clears a control intercept in the specified VMCB.
1460	*
1461	* @returns @c true if the intercept is still set, @c false otherwise.
1462	* @param pVCpu The cross context virtual CPU structure.
1463	* @param pVmcb Pointer to the VM control block.
1464	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1465	*
1466	* @remarks This takes into account if we're executing a nested-guest and only
1467	* removes the control intercept if both the guest -and- nested-guest
1468	* are not intercepting it.
1469	*/
1470	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1471	{
1472	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1473	{
1474	bool fRemove = true;
1475	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1476	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1477	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1478	{
1479	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1480	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1481	}
1482	#else
1483	RT_NOREF(pVCpu);
1484	#endif
1485	if (fRemove)
1486	{
1487	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1488	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1489	}
1490	}
1491
1492	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1493	}
1494
1495
1496	/**
1497	* Exports the guest (or nested-guest) CR0 into the VMCB.
1498	*
1499	* @param pVCpu The cross context virtual CPU structure.
1500	* @param pVmcb Pointer to the VM control block.
1501	*
1502	* @remarks This assumes we always pre-load the guest FPU.
1503	* @remarks No-long-jump zone!!!
1504	*/
1505	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1506	{
1507	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1508
1509	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1510	uint64_t const uGuestCr0 = pCtx->cr0;
1511	uint64_t uShadowCr0 = uGuestCr0;
1512
1513	/* Always enable caching. */
1514	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1515
1516	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1517	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1518	{
1519	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1520	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1521	}
1522
1523	/*
1524	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1525	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1526	* see @bugref{7243#c103}.
1527	*/
1528	if (!(uGuestCr0 & X86_CR0_NE))
1529	{
1530	uShadowCr0 \|= X86_CR0_NE;
1531	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1532	}
1533	else
1534	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1535
1536	/*
1537	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1538	*
1539	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1540	* see @bugref{6944}.
1541	*
1542	* We also don't ever want to honor weird things like cache disable from the guest.
1543	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1544	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1545	*/
1546	if (uShadowCr0 == uGuestCr0)
1547	{
1548	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1549	{
1550	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1551	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1552	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1553	}
1554	else
1555	{
1556	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1557	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1558	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1559	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1560	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1561	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1562	}
1563	}
1564	else
1565	{
1566	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1567	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1568	}
1569	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1570
1571	Assert(!RT_HI_U32(uShadowCr0));
1572	if (pVmcb->guest.u64CR0 != uShadowCr0)
1573	{
1574	pVmcb->guest.u64CR0 = uShadowCr0;
1575	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1576	}
1577	}
1578
1579
1580	/**
1581	* Exports the guest (or nested-guest) CR3 into the VMCB.
1582	*
1583	* @param pVCpu The cross context virtual CPU structure.
1584	* @param pVmcb Pointer to the VM control block.
1585	*
1586	* @remarks No-long-jump zone!!!
1587	*/
1588	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1589	{
1590	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1591
1592	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1593	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1594	if (pVM->hmr0.s.fNestedPaging)
1595	{
1596	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1597	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1598	pVmcb->guest.u64CR3 = pCtx->cr3;
1599	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1600	}
1601	else
1602	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1603
1604	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1605	}
1606
1607
1608	/**
1609	* Exports the guest (or nested-guest) CR4 into the VMCB.
1610	*
1611	* @param pVCpu The cross context virtual CPU structure.
1612	* @param pVmcb Pointer to the VM control block.
1613	*
1614	* @remarks No-long-jump zone!!!
1615	*/
1616	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1617	{
1618	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1619
1620	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1621	uint64_t uShadowCr4 = pCtx->cr4;
1622	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1623	{
1624	switch (pVCpu->hm.s.enmShadowMode)
1625	{
1626	case PGMMODE_REAL:
1627	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1628	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1629
1630	case PGMMODE_32_BIT: /* 32-bit paging. */
1631	uShadowCr4 &= ~X86_CR4_PAE;
1632	break;
1633
1634	case PGMMODE_PAE: /* PAE paging. */
1635	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1636	/** Must use PAE paging as we could use physical memory > 4 GB */
1637	uShadowCr4 \|= X86_CR4_PAE;
1638	break;
1639
1640	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1641	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1642	#ifdef VBOX_WITH_64_BITS_GUESTS
1643	break;
1644	#else
1645	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1646	#endif
1647
1648	default: /* shut up gcc */
1649	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1650	}
1651	}
1652
1653	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1654	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1655	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1656	{
1657	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1658	hmR0SvmUpdateVmRunFunction(pVCpu);
1659	}
1660
1661	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1662	if (uShadowCr4 == pCtx->cr4)
1663	{
1664	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1665	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1666	else
1667	{
1668	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1669	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1670	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1671	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1672	}
1673	}
1674	else
1675	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1676
1677	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1678	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1679	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1680
1681	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1682	Assert(!RT_HI_U32(uShadowCr4));
1683	pVmcb->guest.u64CR4 = uShadowCr4;
1684	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1685
1686	return VINF_SUCCESS;
1687	}
1688
1689
1690	/**
1691	* Exports the guest (or nested-guest) control registers into the VMCB.
1692	*
1693	* @returns VBox status code.
1694	* @param pVCpu The cross context virtual CPU structure.
1695	* @param pVmcb Pointer to the VM control block.
1696	*
1697	* @remarks No-long-jump zone!!!
1698	*/
1699	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1700	{
1701	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1704	{
1705	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1706	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1707
1708	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1709	{
1710	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1711	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1712	}
1713
1714	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1715	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1716
1717	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1718	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1719	{
1720	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1721	if (RT_FAILURE(rc))
1722	return rc;
1723	}
1724
1725	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1726	}
1727	return VINF_SUCCESS;
1728	}
1729
1730
1731	/**
1732	* Exports the guest (or nested-guest) segment registers into the VMCB.
1733	*
1734	* @param pVCpu The cross context virtual CPU structure.
1735	* @param pVmcb Pointer to the VM control block.
1736	*
1737	* @remarks No-long-jump zone!!!
1738	*/
1739	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1740	{
1741	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1742	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1743
1744	/* Guest segment registers. */
1745	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1746	{
1747	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1748	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1749
1750	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1751	{
1752	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1753	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1754	}
1755
1756	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1757	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1758
1759	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1760	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1761
1762	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1763	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1764
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1767
1768	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1769	}
1770
1771	/* Guest TR. */
1772	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1773	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1774
1775	/* Guest LDTR. */
1776	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1777	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1778
1779	/* Guest GDTR. */
1780	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1781	{
1782	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1783	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1784	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1785	}
1786
1787	/* Guest IDTR. */
1788	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1789	{
1790	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1791	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1792	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1793	}
1794
1795	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1796	\| HM_CHANGED_GUEST_TABLE_MASK);
1797	}
1798
1799
1800	/**
1801	* Exports the guest (or nested-guest) MSRs into the VMCB.
1802	*
1803	* @param pVCpu The cross context virtual CPU structure.
1804	* @param pVmcb Pointer to the VM control block.
1805	*
1806	* @remarks No-long-jump zone!!!
1807	*/
1808	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1809	{
1810	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1811	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1812
1813	/* Guest Sysenter MSRs. */
1814	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1815	{
1816	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1817	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1818
1819	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1820	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1821
1822	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1823	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1824	}
1825
1826	/*
1827	* Guest EFER MSR.
1828	* AMD-V requires guest EFER.SVME to be set. Weird.
1829	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1830	*/
1831	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1832	{
1833	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1834	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1835	}
1836
1837	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1838	if ( !CPUMIsGuestInLongModeEx(pCtx)
1839	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1840	{
1841	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1842	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1843	}
1844
1845	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1846	{
1847	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1848	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1849	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1850	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1851	}
1852
1853	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1854	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1855
1856	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1857	\| HM_CHANGED_GUEST_EFER_MSR
1858	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1859	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1860
1861	/*
1862	* Setup the PAT MSR (applicable for Nested Paging only).
1863	*
1864	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1865	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1866	*
1867	* While guests can modify and see the modified values through the shadow values,
1868	* we shall not honor any guest modifications of this MSR to ensure caching is always
1869	* enabled similar to how we clear CR0.CD and NW bits.
1870	*
1871	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1872	*/
1873	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1874
1875	/* Enable the last branch record bit if LBR virtualization is enabled. */
1876	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1877	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1878	}
1879
1880
1881	/**
1882	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1883	* the necessary intercepts accordingly.
1884	*
1885	* @param pVCpu The cross context virtual CPU structure.
1886	* @param pVmcb Pointer to the VM control block.
1887	*
1888	* @remarks No-long-jump zone!!!
1889	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1890	*/
1891	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1892	{
1893	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1894
1895	/** @todo Figure out stepping with nested-guest. */
1896	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1897	{
1898	/*
1899	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1900	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1901	* Instead, they are strictly only requested when the nested hypervisor intercepts
1902	* them -- handled while merging VMCB controls.
1903	*
1904	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1905	* then the nested-guest debug state should be actively loaded on the host so that
1906	* nested-guest reads/writes its own debug registers without causing VM-exits.
1907	*/
1908	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1909	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1910	&& !CPUMIsGuestDebugStateActive(pVCpu))
1911	{
1912	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1913	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1914	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1915	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1916	}
1917
1918	pVmcb->guest.u64DR6 = pCtx->dr[6];
1919	pVmcb->guest.u64DR7 = pCtx->dr[7];
1920	return;
1921	}
1922
1923	/*
1924	* Anyone single stepping on the host side? If so, we'll have to use the
1925	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1926	* the VMM level like the VT-x implementations does.
1927	*/
1928	bool fInterceptMovDRx = false;
1929	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1930	if (fStepping)
1931	{
1932	pVCpu->hmr0.s.fClearTrapFlag = true;
1933	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1934	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1935	}
1936
1937	if ( fStepping
1938	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1939	{
1940	/*
1941	* Use the combined guest and host DRx values found in the hypervisor
1942	* register set because the debugger has breakpoints active or someone
1943	* is single stepping on the host side.
1944	*
1945	* Note! DBGF expects a clean DR6 state before executing guest code.
1946	*/
1947	if (!CPUMIsHyperDebugStateActive(pVCpu))
1948	{
1949	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1950	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1951	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1952	}
1953
1954	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1955	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1956	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1957	{
1958	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1959	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1960	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1961	}
1962
1963	/** @todo If we cared, we could optimize to allow the guest to read registers
1964	* with the same values. */
1965	fInterceptMovDRx = true;
1966	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1967	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1968	}
1969	else
1970	{
1971	/*
1972	* Update DR6, DR7 with the guest values if necessary.
1973	*/
1974	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1975	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1976	{
1977	pVmcb->guest.u64DR7 = pCtx->dr[7];
1978	pVmcb->guest.u64DR6 = pCtx->dr[6];
1979	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1980	}
1981	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1982
1983	/*
1984	* If the guest has enabled debug registers, we need to load them prior to
1985	* executing guest code so they'll trigger at the right time.
1986	*/
1987	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1988	{
1989	if (!CPUMIsGuestDebugStateActive(pVCpu))
1990	{
1991	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1992	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1993	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1994	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1995	}
1996	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
1997	}
1998	/*
1999	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2000	* intercept #DB as DR6 is updated in the VMCB.
2001	*
2002	* Note! If we cared and dared, we could skip intercepting \#DB here.
2003	* However, \#DB shouldn't be performance critical, so we'll play safe
2004	* and keep the code similar to the VT-x code and always intercept it.
2005	*/
2006	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2007	fInterceptMovDRx = true;
2008	}
2009
2010	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2011	if (fInterceptMovDRx)
2012	{
2013	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2014	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2015	{
2016	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2017	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2018	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2019	}
2020	}
2021	else
2022	{
2023	if ( pVmcb->ctrl.u16InterceptRdDRx
2024	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2025	{
2026	pVmcb->ctrl.u16InterceptRdDRx = 0;
2027	pVmcb->ctrl.u16InterceptWrDRx = 0;
2028	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2029	}
2030	}
2031	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2032	}
2033
2034	/**
2035	* Exports the hardware virtualization state into the nested-guest
2036	* VMCB.
2037	*
2038	* @param pVCpu The cross context virtual CPU structure.
2039	* @param pVmcb Pointer to the VM control block.
2040	*
2041	* @remarks No-long-jump zone!!!
2042	*/
2043	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2044	{
2045	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2046
2047	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2048	{
2049	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2050	{
2051	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2052	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2053
2054	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2055	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2056	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2057	NOREF(pVM);
2058
2059	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2060	}
2061
2062	/*
2063	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2064	* since SVM doesn't have a preemption timer.
2065	*
2066	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2067	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2068	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2069	*/
2070	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2071	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2072	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2073	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2074	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2075	{
2076	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2077	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2078	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2079	}
2080	else
2081	{
2082	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2083	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2084	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2085	}
2086	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2087
2088	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2089	}
2090	}
2091
2092
2093	/**
2094	* Exports the guest APIC TPR state into the VMCB.
2095	*
2096	* @returns VBox status code.
2097	* @param pVCpu The cross context virtual CPU structure.
2098	* @param pVmcb Pointer to the VM control block.
2099	*/
2100	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2101	{
2102	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2103
2104	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2105	{
2106	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2107	if ( PDMHasApic(pVM)
2108	&& APICIsEnabled(pVCpu))
2109	{
2110	bool fPendingIntr;
2111	uint8_t u8Tpr;
2112	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2113	AssertRCReturn(rc, rc);
2114
2115	/* Assume that we need to trap all TPR accesses and thus need not check on
2116	every #VMEXIT if we should update the TPR. */
2117	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2118	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2119
2120	if (!pVM->hm.s.fTprPatchingActive)
2121	{
2122	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2123	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2124
2125	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2126	can deliver the interrupt to the guest. */
2127	if (fPendingIntr)
2128	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2129	else
2130	{
2131	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2132	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2133	}
2134
2135	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2136	}
2137	else
2138	{
2139	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2140	pVmcb->guest.u64LSTAR = u8Tpr;
2141	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2142
2143	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2144	if (fPendingIntr)
2145	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2146	else
2147	{
2148	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2149	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2150	}
2151	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2152	}
2153	}
2154	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2155	}
2156	return VINF_SUCCESS;
2157	}
2158
2159
2160	/**
2161	* Sets up the exception interrupts required for guest execution in the VMCB.
2162	*
2163	* @param pVCpu The cross context virtual CPU structure.
2164	* @param pVmcb Pointer to the VM control block.
2165	*
2166	* @remarks No-long-jump zone!!!
2167	*/
2168	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2169	{
2170	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2171
2172	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2173	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2174	{
2175	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2176	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2177	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2178	else
2179	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2180
2181	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2182	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2183	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2184	else
2185	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2186
2187	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2188	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2189	}
2190	}
2191
2192
2193	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2194	/**
2195	* Merges guest and nested-guest intercepts for executing the nested-guest using
2196	* hardware-assisted SVM.
2197	*
2198	* This merges the guest and nested-guest intercepts in a way that if the outer
2199	* guest intercept is set we need to intercept it in the nested-guest as
2200	* well.
2201	*
2202	* @param pVCpu The cross context virtual CPU structure.
2203	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2204	*/
2205	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2206	{
2207	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2208	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2209	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2210	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2211
2212	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2213	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2214	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2215
2216	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2217	XCR0 syncing during worlds switching). */
2218	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2219
2220	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2221	if (!pVM->hmr0.s.fNestedPaging)
2222	{
2223	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2224	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2225	}
2226
2227	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2228	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2229	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2230
2231	/*
2232	* Merge the guest's exception intercepts into the nested-guest VMCB.
2233	*
2234	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2235	* while executing the nested-guest.
2236	*
2237	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2238	* be tweaked later depending on how we wish to implement breakpoints.
2239	*
2240	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2241	* loaded into their guests, not ours.
2242	*
2243	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2244	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2245	*/
2246	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2247	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2248	& ~( RT_BIT(X86_XCPT_UD)
2249	\| RT_BIT(X86_XCPT_BP)
2250	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2251	#else
2252	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2253	#endif
2254
2255	/*
2256	* Adjust intercepts while executing the nested-guest that differ from the
2257	* outer guest intercepts.
2258	*
2259	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2260	* that belong to the nested-guest to the outer guest.
2261	*
2262	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2263	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2264	*/
2265	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2266	\| SVM_CTRL_INTERCEPT_VMMCALL))
2267	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2268
2269	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2270	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2271
2272	/* Finally, update the VMCB clean bits. */
2273	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2274	}
2275	#endif
2276
2277
2278	/**
2279	* Enters the AMD-V session.
2280	*
2281	* @returns VBox status code.
2282	* @param pVCpu The cross context virtual CPU structure.
2283	*/
2284	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2285	{
2286	AssertPtr(pVCpu);
2287	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2288	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2289
2290	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2291	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2292	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2293
2294	pVCpu->hmr0.s.fLeaveDone = false;
2295	return VINF_SUCCESS;
2296	}
2297
2298
2299	/**
2300	* Thread-context callback for AMD-V.
2301	*
2302	* This is used together with RTThreadCtxHookCreate() on platforms which
2303	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2304	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2305	*
2306	* @param enmEvent The thread-context event.
2307	* @param pVCpu The cross context virtual CPU structure.
2308	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2309	* @thread EMT(pVCpu)
2310	*/
2311	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2312	{
2313	NOREF(fGlobalInit);
2314
2315	switch (enmEvent)
2316	{
2317	case RTTHREADCTXEVENT_OUT:
2318	{
2319	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2320	VMCPU_ASSERT_EMT(pVCpu);
2321
2322	/* No longjmps (log-flush, locks) in this fragile context. */
2323	VMMRZCallRing3Disable(pVCpu);
2324
2325	if (!pVCpu->hmr0.s.fLeaveDone)
2326	{
2327	hmR0SvmLeave(pVCpu, false /* fImportState */);
2328	pVCpu->hmr0.s.fLeaveDone = true;
2329	}
2330
2331	/* Leave HM context, takes care of local init (term). */
2332	int rc = HMR0LeaveCpu(pVCpu);
2333	AssertRC(rc); NOREF(rc);
2334
2335	/* Restore longjmp state. */
2336	VMMRZCallRing3Enable(pVCpu);
2337	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2338	break;
2339	}
2340
2341	case RTTHREADCTXEVENT_IN:
2342	{
2343	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2344	VMCPU_ASSERT_EMT(pVCpu);
2345
2346	/* No longjmps (log-flush, locks) in this fragile context. */
2347	VMMRZCallRing3Disable(pVCpu);
2348
2349	/*
2350	* Initialize the bare minimum state required for HM. This takes care of
2351	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2352	*/
2353	int rc = hmR0EnterCpu(pVCpu);
2354	AssertRC(rc); NOREF(rc);
2355	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2356	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2357
2358	pVCpu->hmr0.s.fLeaveDone = false;
2359
2360	/* Restore longjmp state. */
2361	VMMRZCallRing3Enable(pVCpu);
2362	break;
2363	}
2364
2365	default:
2366	break;
2367	}
2368	}
2369
2370
2371	/**
2372	* Saves the host state.
2373	*
2374	* @returns VBox status code.
2375	* @param pVCpu The cross context virtual CPU structure.
2376	*
2377	* @remarks No-long-jump zone!!!
2378	*/
2379	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2380	{
2381	NOREF(pVCpu);
2382
2383	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2384	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2385	return VINF_SUCCESS;
2386	}
2387
2388
2389	/**
2390	* Exports the guest or nested-guest state from the virtual-CPU context into the
2391	* VMCB.
2392	*
2393	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2394	* code based on the virtual-CPU mode.
2395	*
2396	* @returns VBox status code.
2397	* @param pVCpu The cross context virtual CPU structure.
2398	* @param pSvmTransient Pointer to the SVM-transient structure.
2399	*
2400	* @remarks No-long-jump zone!!!
2401	*/
2402	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2403	{
2404	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2405
2406	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2407	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2408	Assert(pVmcb);
2409
2410	pVmcb->guest.u64RIP = pCtx->rip;
2411	pVmcb->guest.u64RSP = pCtx->rsp;
2412	pVmcb->guest.u64RFlags = pCtx->eflags.u;
2413	pVmcb->guest.u64RAX = pCtx->rax;
2414
2415	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2416	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2417
2418	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2419	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2420	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2421	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2422	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2423
2424	ASMSetFlags(fEFlags);
2425
2426	if (!fIsNestedGuest)
2427	{
2428	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2429	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2430	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2431	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2432	}
2433
2434	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2435	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2436	\| HM_CHANGED_GUEST_RFLAGS
2437	\| HM_CHANGED_GUEST_GPRS_MASK
2438	\| HM_CHANGED_GUEST_X87
2439	\| HM_CHANGED_GUEST_SSE_AVX
2440	\| HM_CHANGED_GUEST_OTHER_XSAVE
2441	\| HM_CHANGED_GUEST_XCRx
2442	\| HM_CHANGED_GUEST_TSC_AUX
2443	\| HM_CHANGED_GUEST_OTHER_MSRS;
2444	if (fIsNestedGuest)
2445	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2446	\| HM_CHANGED_GUEST_APIC_TPR;
2447
2448	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2449	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2450
2451	#ifdef VBOX_STRICT
2452	/*
2453	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2454	* except for the host-context and/or shared host-guest context bits.
2455	*/
2456	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2457	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2458	("fCtxChanged=%#RX64\n", fCtxChanged));
2459
2460	/*
2461	* If we need to log state that isn't always imported, we'll need to import them here.
2462	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2463	*/
2464	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2465	#endif
2466
2467	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2468	return VINF_SUCCESS;
2469	}
2470
2471	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2472
2473	/**
2474	* Merges the guest and nested-guest MSR permission bitmap.
2475	*
2476	* If the guest is intercepting an MSR we need to intercept it regardless of
2477	* whether the nested-guest is intercepting it or not.
2478	*
2479	* @param pHostCpu The HM physical-CPU structure.
2480	* @param pVCpu The cross context virtual CPU structure.
2481	*
2482	* @remarks No-long-jmp zone!!!
2483	*/
2484	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2485	{
2486	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2487	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2488	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2489
2490	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2491	uint32_t const offRsvdQwords = 0x1800 >> 3;
2492	for (uint32_t i = 0; i < offRsvdQwords; i++)
2493	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2494	}
2495
2496
2497	/**
2498	* Caches the nested-guest VMCB fields before we modify them for execution using
2499	* hardware-assisted SVM.
2500	*
2501	* @returns true if the VMCB was previously already cached, false otherwise.
2502	* @param pVCpu The cross context virtual CPU structure.
2503	*
2504	* @sa HMNotifySvmNstGstVmexit.
2505	*/
2506	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2507	{
2508	/*
2509	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2510	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2511	*
2512	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2513	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2514	*/
2515	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2516	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2517	if (!fWasCached)
2518	{
2519	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2520	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2521	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2522	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2523	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2524	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2525	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2526	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2527	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2528	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2529	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2530	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2531	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2532	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2533	pVmcbNstGstCache->fCacheValid = true;
2534	Log4Func(("Cached VMCB fields\n"));
2535	}
2536
2537	return fWasCached;
2538	}
2539
2540
2541	/**
2542	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2543	*
2544	* This is done the first time we enter nested-guest execution using SVM R0
2545	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2546	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2547	*
2548	* @param pVCpu The cross context virtual CPU structure.
2549	*/
2550	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2551	{
2552	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2553	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2554
2555	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2556
2557	/*
2558	* First cache the nested-guest VMCB fields we may potentially modify.
2559	*/
2560	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2561	if (!fVmcbCached)
2562	{
2563	/*
2564	* The IOPM of the nested-guest can be ignored because the the guest always
2565	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2566	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2567	*/
2568	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2569
2570	/*
2571	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2572	* nested-paging suddenly while executing a VM (see assertion at the end of
2573	* Trap0eHandler() in PGMAllBth.h).
2574	*/
2575	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2576
2577	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2578	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2579
2580	/*
2581	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2582	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2583	*/
2584	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2585
2586	# ifdef DEBUG_ramshankar
2587	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2588	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2589	# endif
2590
2591	/*
2592	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2593	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2594	*/
2595	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2596	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2597	\| SVM_CTRL_INTERCEPT_VMLOAD;
2598
2599	/*
2600	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2601	* CLGI/STGI instructions executed by the nested-guest.
2602	*/
2603	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2604	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2605	\| SVM_CTRL_INTERCEPT_STGI;
2606
2607	/* Merge the guest and nested-guest intercepts. */
2608	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2609
2610	/* Update the VMCB clean bits. */
2611	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2612	}
2613	else
2614	{
2615	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2616	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2617	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2618	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2619	}
2620	}
2621
2622	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2623
2624	/**
2625	* Exports the state shared between the host and guest (or nested-guest) into
2626	* the VMCB.
2627	*
2628	* @param pVCpu The cross context virtual CPU structure.
2629	* @param pVmcb Pointer to the VM control block.
2630	*
2631	* @remarks No-long-jump zone!!!
2632	*/
2633	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2634	{
2635	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2636	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2637
2638	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2639	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2640
2641	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2642	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2643	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2644	}
2645
2646
2647	/**
2648	* Worker for SVMR0ImportStateOnDemand.
2649	*
2650	* @param pVCpu The cross context virtual CPU structure.
2651	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2652	*/
2653	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2654	{
2655	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2656
2657	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2658	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2659	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2660	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2661
2662	/*
2663	* We disable interrupts to make the updating of the state and in particular
2664	* the fExtrn modification atomic wrt to preemption hooks.
2665	*/
2666	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2667
2668	fWhat &= pCtx->fExtrn;
2669	if (fWhat)
2670	{
2671	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2672	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2673	{
2674	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2675	{
2676	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2677	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2678	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2679	}
2680	}
2681
2682	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2683	{
2684	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2685	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2686	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2687	}
2688	#endif
2689
2690	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2691	CPUMUpdateInterruptShadowEx(pCtx, pVmcbCtrl->IntShadow.n.u1IntShadow, pVmcbGuest->u64RIP);
2692
2693	if (fWhat & CPUMCTX_EXTRN_RIP)
2694	pCtx->rip = pVmcbGuest->u64RIP;
2695
2696	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2697	{
2698	pCtx->eflags.u = pVmcbGuest->u64RFlags;
2699	if (pVCpu->hmr0.s.fClearTrapFlag)
2700	{
2701	pVCpu->hmr0.s.fClearTrapFlag = false;
2702	pCtx->eflags.Bits.u1TF = 0;
2703	}
2704	}
2705
2706	if (fWhat & CPUMCTX_EXTRN_RSP)
2707	pCtx->rsp = pVmcbGuest->u64RSP;
2708
2709	if (fWhat & CPUMCTX_EXTRN_RAX)
2710	pCtx->rax = pVmcbGuest->u64RAX;
2711
2712	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2713	{
2714	if (fWhat & CPUMCTX_EXTRN_CS)
2715	{
2716	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2717	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2718	/** @todo SELM might need to be fixed as it too should not care about the
2719	* granularity bit. See @bugref{6785}. */
2720	if ( !pCtx->cs.Attr.n.u1Granularity
2721	&& pCtx->cs.Attr.n.u1Present
2722	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2723	{
2724	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2725	pCtx->cs.Attr.n.u1Granularity = 1;
2726	}
2727	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2728	}
2729	if (fWhat & CPUMCTX_EXTRN_SS)
2730	{
2731	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2732	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2733	/*
2734	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2735	* VMCB and uses that and thus it's possible that when the CPL changes during
2736	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2737	* AMD Fusion CPUs with 64-bit guests.
2738	*
2739	* See AMD spec. 15.5.1 "Basic operation".
2740	*/
2741	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2742	uint8_t const uCpl = pVmcbGuest->u8CPL;
2743	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2744	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2745	}
2746	if (fWhat & CPUMCTX_EXTRN_DS)
2747	{
2748	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2749	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2750	}
2751	if (fWhat & CPUMCTX_EXTRN_ES)
2752	{
2753	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2754	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2755	}
2756	if (fWhat & CPUMCTX_EXTRN_FS)
2757	{
2758	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2759	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2760	}
2761	if (fWhat & CPUMCTX_EXTRN_GS)
2762	{
2763	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2764	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2765	}
2766	}
2767
2768	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2769	{
2770	if (fWhat & CPUMCTX_EXTRN_TR)
2771	{
2772	/*
2773	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2774	* are used between Intel and AMD, see @bugref{6208#c39}.
2775	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2776	*/
2777	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2778	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2779	{
2780	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2781	\|\| CPUMIsGuestInLongModeEx(pCtx))
2782	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2783	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2784	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2785	}
2786	}
2787
2788	if (fWhat & CPUMCTX_EXTRN_LDTR)
2789	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2790
2791	if (fWhat & CPUMCTX_EXTRN_GDTR)
2792	{
2793	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2794	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2795	}
2796
2797	if (fWhat & CPUMCTX_EXTRN_IDTR)
2798	{
2799	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2800	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2801	}
2802	}
2803
2804	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2805	{
2806	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2807	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2808	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2809	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2810	}
2811
2812	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2813	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2814	{
2815	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2816	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2817	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2818	}
2819
2820	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2821	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2822
2823	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2824	{
2825	if (fWhat & CPUMCTX_EXTRN_DR6)
2826	{
2827	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2828	pCtx->dr[6] = pVmcbGuest->u64DR6;
2829	else
2830	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2831	}
2832
2833	if (fWhat & CPUMCTX_EXTRN_DR7)
2834	{
2835	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2836	pCtx->dr[7] = pVmcbGuest->u64DR7;
2837	else
2838	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2839	}
2840	}
2841
2842	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2843	{
2844	if (fWhat & CPUMCTX_EXTRN_CR0)
2845	{
2846	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2847	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2848	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2849	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2850	CPUMSetGuestCR0(pVCpu, uCr0);
2851	VMMRZCallRing3Enable(pVCpu);
2852	}
2853
2854	if (fWhat & CPUMCTX_EXTRN_CR2)
2855	pCtx->cr2 = pVmcbGuest->u64CR2;
2856
2857	if (fWhat & CPUMCTX_EXTRN_CR3)
2858	{
2859	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2860	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2861	{
2862	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2863	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2864	}
2865	}
2866
2867	/* Changes to CR4 are always intercepted. */
2868	}
2869
2870	/* Update fExtrn. */
2871	pCtx->fExtrn &= ~fWhat;
2872
2873	/* If everything has been imported, clear the HM keeper bit. */
2874	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2875	{
2876	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2877	Assert(!pCtx->fExtrn);
2878	}
2879	}
2880	else
2881	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2882
2883	ASMSetFlags(fEFlags);
2884
2885	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2886
2887	/*
2888	* Honor any pending CR3 updates.
2889	*
2890	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2891	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2892	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2893	* handling -> hmR0SvmImportGuestState() and here we are.
2894	*
2895	* The reason for such complicated handling is because VM-exits that call into PGM expect
2896	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2897	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2898	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2899	*
2900	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2901	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2902	*/
2903	if ( VMMRZCallRing3IsEnabled(pVCpu)
2904	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2905	{
2906	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2907	PGMUpdateCR3(pVCpu, pCtx->cr3);
2908	}
2909	}
2910
2911
2912	/**
2913	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2914	* context.
2915	*
2916	* Currently there is no residual state left in the CPU that is not updated in the
2917	* VMCB.
2918	*
2919	* @returns VBox status code.
2920	* @param pVCpu The cross context virtual CPU structure.
2921	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2922	*/
2923	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2924	{
2925	hmR0SvmImportGuestState(pVCpu, fWhat);
2926	return VINF_SUCCESS;
2927	}
2928
2929
2930	/**
2931	* Gets SVM \#VMEXIT auxiliary information.
2932	*
2933	* @returns VBox status code.
2934	* @param pVCpu The cross context virtual CPU structure.
2935	* @param pSvmExitAux Where to store the auxiliary info.
2936	*/
2937	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2938	{
2939	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2940	if (RT_LIKELY(pSvmTransient))
2941	{
2942	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2943	if (RT_LIKELY(pVmcb))
2944	{
2945	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2946	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2947	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2948	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2949	return VINF_SUCCESS;
2950	}
2951	return VERR_SVM_IPE_5;
2952	}
2953	return VERR_NOT_AVAILABLE;
2954	}
2955
2956
2957	/**
2958	* Does the necessary state syncing before returning to ring-3 for any reason
2959	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2960	*
2961	* @param pVCpu The cross context virtual CPU structure.
2962	* @param fImportState Whether to import the guest state from the VMCB back
2963	* to the guest-CPU context.
2964	*
2965	* @remarks No-long-jmp zone!!!
2966	*/
2967	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2968	{
2969	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2970	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2971
2972	/*
2973	* !!! IMPORTANT !!!
2974	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2975	*/
2976
2977	/* Save the guest state if necessary. */
2978	if (fImportState)
2979	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2980
2981	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2982	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2983	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2984
2985	/*
2986	* Restore host debug registers if necessary and resync on next R0 reentry.
2987	*/
2988	#ifdef VBOX_STRICT
2989	if (CPUMIsHyperDebugStateActive(pVCpu))
2990	{
2991	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2992	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2993	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2994	}
2995	#endif
2996	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2997	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2998	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2999
3000	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3001	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3002	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3004	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3005	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3006	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3007
3008	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3009	}
3010
3011
3012	/**
3013	* Leaves the AMD-V session.
3014	*
3015	* Only used while returning to ring-3 either due to longjump or exits to
3016	* ring-3.
3017	*
3018	* @returns VBox status code.
3019	* @param pVCpu The cross context virtual CPU structure.
3020	*/
3021	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3022	{
3023	HM_DISABLE_PREEMPT(pVCpu);
3024	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3025	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3026
3027	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3028	and done this from the SVMR0ThreadCtxCallback(). */
3029	if (!pVCpu->hmr0.s.fLeaveDone)
3030	{
3031	hmR0SvmLeave(pVCpu, true /* fImportState */);
3032	pVCpu->hmr0.s.fLeaveDone = true;
3033	}
3034
3035	/*
3036	* !!! IMPORTANT !!!
3037	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3038	*/
3039
3040	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3041	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3042	VMMR0ThreadCtxHookDisable(pVCpu);
3043
3044	/* Leave HM context. This takes care of local init (term). */
3045	int rc = HMR0LeaveCpu(pVCpu);
3046
3047	HM_RESTORE_PREEMPT();
3048	return rc;
3049	}
3050
3051
3052	/**
3053	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3054	* any remaining host state) before we go back to ring-3 due to an assertion.
3055	*
3056	* @param pVCpu The cross context virtual CPU structure.
3057	*/
3058	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3059	{
3060	/*
3061	* !!! IMPORTANT !!!
3062	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3063	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3064	*/
3065	VMMR0AssertionRemoveNotification(pVCpu);
3066	VMMRZCallRing3Disable(pVCpu);
3067	HM_DISABLE_PREEMPT(pVCpu);
3068
3069	/* Import the entire guest state. */
3070	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3071
3072	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3073	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3074
3075	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3076	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3077
3078	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3079	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3080	VMMR0ThreadCtxHookDisable(pVCpu);
3081
3082	/* Leave HM context. This takes care of local init (term). */
3083	HMR0LeaveCpu(pVCpu);
3084
3085	HM_RESTORE_PREEMPT();
3086	return VINF_SUCCESS;
3087	}
3088
3089
3090	/**
3091	* Take necessary actions before going back to ring-3.
3092	*
3093	* An action requires us to go back to ring-3. This function does the necessary
3094	* steps before we can safely return to ring-3. This is not the same as longjmps
3095	* to ring-3, this is voluntary.
3096	*
3097	* @returns Strict VBox status code.
3098	* @param pVCpu The cross context virtual CPU structure.
3099	* @param rcExit The reason for exiting to ring-3. Can be
3100	* VINF_VMM_UNKNOWN_RING3_CALL.
3101	*/
3102	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3103	{
3104	Assert(pVCpu);
3105	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3106
3107	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3108	VMMRZCallRing3Disable(pVCpu);
3109	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3110	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3111
3112	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3113	if (pVCpu->hm.s.Event.fPending)
3114	{
3115	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3116	Assert(!pVCpu->hm.s.Event.fPending);
3117	}
3118
3119	/* Sync. the necessary state for going back to ring-3. */
3120	hmR0SvmLeaveSession(pVCpu);
3121	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3122
3123	/* Thread-context hooks are unregistered at this point!!! */
3124	/* Ring-3 callback notifications are unregistered at this point!!! */
3125
3126	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3127	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3128	\| CPUM_CHANGED_LDTR
3129	\| CPUM_CHANGED_GDTR
3130	\| CPUM_CHANGED_IDTR
3131	\| CPUM_CHANGED_TR
3132	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3133	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3134	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3135	{
3136	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3137	}
3138
3139	/* Update the exit-to-ring 3 reason. */
3140	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3141
3142	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3143	if ( rcExit != VINF_EM_RAW_INTERRUPT
3144	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3145	{
3146	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3147	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3148	}
3149
3150	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3151	VMMRZCallRing3Enable(pVCpu);
3152
3153	/*
3154	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3155	* and if we're injecting an event we should have a TRPM trap pending.
3156	*/
3157	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3158	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3159	VERR_SVM_IPE_5);
3160	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3161	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3162	VERR_SVM_IPE_4);
3163
3164	return rcExit;
3165	}
3166
3167
3168	/**
3169	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3170	* intercepts.
3171	*
3172	* @param pVCpu The cross context virtual CPU structure.
3173	* @param pVmcb Pointer to the VM control block.
3174	*
3175	* @remarks No-long-jump zone!!!
3176	*/
3177	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3178	{
3179	/*
3180	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3181	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3182	* RDTSC/RDTSCP as well.
3183	*/
3184	bool fParavirtTsc;
3185	uint64_t uTscOffset;
3186	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3187
3188	bool fIntercept;
3189	if (fCanUseRealTsc)
3190	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3191	else
3192	{
3193	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3194	fIntercept = true;
3195	}
3196
3197	if (!fIntercept)
3198	{
3199	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3200	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3201	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3202	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3203	#endif
3204
3205	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3206	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3207	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3208	}
3209
3210	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3211	information before every VM-entry, hence we have nothing to do here at the moment. */
3212	if (fParavirtTsc)
3213	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3214	}
3215
3216
3217	/**
3218	* Sets an event as a pending event to be injected into the guest.
3219	*
3220	* @param pVCpu The cross context virtual CPU structure.
3221	* @param pEvent Pointer to the SVM event.
3222	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3223	* page-fault.
3224	*
3225	* @remarks Statistics counter assumes this is a guest event being reflected to
3226	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3227	*/
3228	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3229	{
3230	Assert(!pVCpu->hm.s.Event.fPending);
3231	Assert(pEvent->n.u1Valid);
3232
3233	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3234	pVCpu->hm.s.Event.fPending = true;
3235	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3236
3237	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3238	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3239	}
3240
3241
3242	/**
3243	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3244	*
3245	* @param pVCpu The cross context virtual CPU structure.
3246	*/
3247	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3248	{
3249	SVMEVENT Event;
3250	Event.u = 0;
3251	Event.n.u1Valid = 1;
3252	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3253	Event.n.u8Vector = X86_XCPT_DE;
3254	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3255	}
3256
3257
3258	/**
3259	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3260	*
3261	* @param pVCpu The cross context virtual CPU structure.
3262	*/
3263	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3264	{
3265	SVMEVENT Event;
3266	Event.u = 0;
3267	Event.n.u1Valid = 1;
3268	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3269	Event.n.u8Vector = X86_XCPT_UD;
3270	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3271	}
3272
3273
3274	/**
3275	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3276	*
3277	* @param pVCpu The cross context virtual CPU structure.
3278	*/
3279	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3280	{
3281	SVMEVENT Event;
3282	Event.u = 0;
3283	Event.n.u1Valid = 1;
3284	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3285	Event.n.u8Vector = X86_XCPT_DB;
3286	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3287	}
3288
3289
3290	/**
3291	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3292	*
3293	* @param pVCpu The cross context virtual CPU structure.
3294	* @param u32ErrCode The error-code for the page-fault.
3295	* @param uFaultAddress The page fault address (CR2).
3296	*
3297	* @remarks This updates the guest CR2 with @a uFaultAddress!
3298	*/
3299	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3300	{
3301	SVMEVENT Event;
3302	Event.u = 0;
3303	Event.n.u1Valid = 1;
3304	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3305	Event.n.u8Vector = X86_XCPT_PF;
3306	Event.n.u1ErrorCodeValid = 1;
3307	Event.n.u32ErrorCode = u32ErrCode;
3308
3309	/* Update CR2 of the guest. */
3310	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3311	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3312	{
3313	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3314	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3315	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3316	}
3317
3318	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3319	}
3320
3321
3322	/**
3323	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3324	*
3325	* @param pVCpu The cross context virtual CPU structure.
3326	*/
3327	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3328	{
3329	SVMEVENT Event;
3330	Event.u = 0;
3331	Event.n.u1Valid = 1;
3332	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3333	Event.n.u8Vector = X86_XCPT_MF;
3334	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3335	}
3336
3337
3338	/**
3339	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3340	*
3341	* @param pVCpu The cross context virtual CPU structure.
3342	*/
3343	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3344	{
3345	SVMEVENT Event;
3346	Event.u = 0;
3347	Event.n.u1Valid = 1;
3348	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3349	Event.n.u8Vector = X86_XCPT_DF;
3350	Event.n.u1ErrorCodeValid = 1;
3351	Event.n.u32ErrorCode = 0;
3352	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3353	}
3354
3355
3356	/**
3357	* Injects an event into the guest upon VMRUN by updating the relevant field
3358	* in the VMCB.
3359	*
3360	* @param pVCpu The cross context virtual CPU structure.
3361	* @param pVmcb Pointer to the guest VM control block.
3362	* @param pEvent Pointer to the event.
3363	*
3364	* @remarks No-long-jump zone!!!
3365	* @remarks Requires CR0!
3366	*/
3367	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3368	{
3369	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3370	pVmcb->ctrl.EventInject.u = pEvent->u;
3371	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3372	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3373	{
3374	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3375	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3376	}
3377	else
3378	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3379	RT_NOREF(pVCpu);
3380
3381	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3382	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3383	}
3384
3385
3386
3387	/**
3388	* Converts any TRPM trap into a pending HM event. This is typically used when
3389	* entering from ring-3 (not longjmp returns).
3390	*
3391	* @param pVCpu The cross context virtual CPU structure.
3392	*/
3393	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3394	{
3395	Assert(TRPMHasTrap(pVCpu));
3396	Assert(!pVCpu->hm.s.Event.fPending);
3397
3398	uint8_t uVector;
3399	TRPMEVENT enmTrpmEvent;
3400	uint32_t uErrCode;
3401	RTGCUINTPTR GCPtrFaultAddress;
3402	uint8_t cbInstr;
3403
3404	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3405	AssertRC(rc);
3406
3407	SVMEVENT Event;
3408	Event.u = 0;
3409	Event.n.u1Valid = 1;
3410	Event.n.u8Vector = uVector;
3411
3412	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3413	if (enmTrpmEvent == TRPM_TRAP)
3414	{
3415	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3416	switch (uVector)
3417	{
3418	case X86_XCPT_NMI:
3419	{
3420	Event.n.u3Type = SVM_EVENT_NMI;
3421	break;
3422	}
3423
3424	case X86_XCPT_BP:
3425	case X86_XCPT_OF:
3426	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3427	RT_FALL_THRU();
3428
3429	case X86_XCPT_PF:
3430	case X86_XCPT_DF:
3431	case X86_XCPT_TS:
3432	case X86_XCPT_NP:
3433	case X86_XCPT_SS:
3434	case X86_XCPT_GP:
3435	case X86_XCPT_AC:
3436	{
3437	Event.n.u1ErrorCodeValid = 1;
3438	Event.n.u32ErrorCode = uErrCode;
3439	break;
3440	}
3441	}
3442	}
3443	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3444	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3445	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3446	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3447	else
3448	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3449
3450	rc = TRPMResetTrap(pVCpu);
3451	AssertRC(rc);
3452
3453	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3454	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3455
3456	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3457	}
3458
3459
3460	/**
3461	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3462	* AMD-V to execute any instruction.
3463	*
3464	* @param pVCpu The cross context virtual CPU structure.
3465	*/
3466	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3467	{
3468	Assert(pVCpu->hm.s.Event.fPending);
3469	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3470
3471	SVMEVENT Event;
3472	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3473
3474	uint8_t uVector = Event.n.u8Vector;
3475	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3476
3477	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3478
3479	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3480	AssertRC(rc);
3481
3482	if (Event.n.u1ErrorCodeValid)
3483	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3484
3485	if ( enmTrapType == TRPM_TRAP
3486	&& uVector == X86_XCPT_PF)
3487	{
3488	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3489	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3490	}
3491	else if (enmTrapType == TRPM_SOFTWARE_INT)
3492	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3493	pVCpu->hm.s.Event.fPending = false;
3494	}
3495
3496
3497	/**
3498	* Sets the virtual interrupt intercept control in the VMCB.
3499	*
3500	* @param pVCpu The cross context virtual CPU structure.
3501	* @param pVmcb Pointer to the VM control block.
3502	*/
3503	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3504	{
3505	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3506
3507	/*
3508	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3509	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3510	* (updating ISR at the right time) and inject the interrupt.
3511	*
3512	* With AVIC is supported, we could make use of the asynchronously delivery without
3513	* #VMEXIT and we would be passing the AVIC page to SVM.
3514	*
3515	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3516	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3517	*/
3518	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3519	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3520	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3521	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3522	Log4(("Set VINTR intercept\n"));
3523	}
3524
3525
3526	/**
3527	* Clears the virtual interrupt intercept control in the VMCB as
3528	* we are figured the guest is unable process any interrupts
3529	* at this point of time.
3530	*
3531	* @param pVCpu The cross context virtual CPU structure.
3532	* @param pVmcb Pointer to the VM control block.
3533	*/
3534	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3535	{
3536	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3537
3538	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3539	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3540	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3541	{
3542	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3543	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3544	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3545	Log4(("Cleared VINTR intercept\n"));
3546	}
3547	}
3548
3549
3550	/**
3551	* Evaluates the event to be delivered to the guest and sets it as the pending
3552	* event.
3553	*
3554	* @returns Strict VBox status code.
3555	* @param pVCpu The cross context virtual CPU structure.
3556	* @param pSvmTransient Pointer to the SVM transient structure.
3557	*/
3558	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3559	{
3560	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3561	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3562	\| CPUMCTX_EXTRN_RFLAGS
3563	\| CPUMCTX_EXTRN_INHIBIT_INT
3564	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3565
3566	Assert(!pVCpu->hm.s.Event.fPending);
3567	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3568	Assert(pVmcb);
3569
3570	bool const fGif = CPUMGetGuestGif(pCtx);
3571	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(pCtx);
3572	bool const fBlockNmi = CPUMAreInterruptsInhibitedByNmi(pCtx);
3573
3574	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3575	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3576	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3577
3578	/** @todo SMI. SMIs take priority over NMIs. */
3579
3580	/*
3581	* Check if the guest or nested-guest can receive NMIs.
3582	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3583	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3584	*/
3585	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3586	&& !fBlockNmi)
3587	{
3588	if ( fGif
3589	&& !fIntShadow)
3590	{
3591	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3592	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3593	{
3594	Log4(("Intercepting NMI -> #VMEXIT\n"));
3595	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3596	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3597	}
3598	#endif
3599	Log4(("Setting NMI pending for injection\n"));
3600	SVMEVENT Event;
3601	Event.u = 0;
3602	Event.n.u1Valid = 1;
3603	Event.n.u8Vector = X86_XCPT_NMI;
3604	Event.n.u3Type = SVM_EVENT_NMI;
3605	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3606	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3607	}
3608	else if (!fGif)
3609	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3610	else if (!pSvmTransient->fIsNestedGuest)
3611	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3612	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3613	}
3614	/*
3615	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3616	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3617	* it from the APIC device.
3618	*
3619	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3620	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3621	* do that work when we execute nested-guest code esp. since all the required information
3622	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3623	* the virtual interrupt controller.
3624	*
3625	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3626	*/
3627	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3628	&& !pVCpu->hm.s.fSingleInstruction)
3629	{
3630	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u & X86_EFL_IF)
3631	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3632	if ( fGif
3633	&& !fBlockInt
3634	&& !fIntShadow)
3635	{
3636	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3637	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3638	{
3639	Log4(("Intercepting INTR -> #VMEXIT\n"));
3640	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3641	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3642	}
3643	#endif
3644	uint8_t u8Interrupt;
3645	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3646	if (RT_SUCCESS(rc))
3647	{
3648	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3649	SVMEVENT Event;
3650	Event.u = 0;
3651	Event.n.u1Valid = 1;
3652	Event.n.u8Vector = u8Interrupt;
3653	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3654	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3655	}
3656	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3657	{
3658	/*
3659	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3660	* updated eventually when the TPR is written by the guest.
3661	*/
3662	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3663	}
3664	else
3665	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3666	}
3667	else if (!fGif)
3668	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3669	else if (!pSvmTransient->fIsNestedGuest)
3670	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3671	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3672	}
3673
3674	return VINF_SUCCESS;
3675	}
3676
3677
3678	/**
3679	* Injects any pending events into the guest (or nested-guest).
3680	*
3681	* @param pVCpu The cross context virtual CPU structure.
3682	* @param pVmcb Pointer to the VM control block.
3683	*
3684	* @remarks Must only be called when we are guaranteed to enter
3685	* hardware-assisted SVM execution and not return to ring-3
3686	* prematurely.
3687	*/
3688	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3689	{
3690	Assert(!TRPMHasTrap(pVCpu));
3691	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3692
3693	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx);
3694	#ifdef VBOX_STRICT
3695	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3696	bool const fGif = CPUMGetGuestGif(pCtx);
3697	bool fAllowInt = fGif;
3698	if (fGif)
3699	{
3700	/*
3701	* For nested-guests we have no way to determine if we're injecting a physical or
3702	* virtual interrupt at this point. Hence the partial verification below.
3703	*/
3704	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3705	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3706	else
3707	fAllowInt = RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
3708	}
3709	#endif
3710
3711	if (pVCpu->hm.s.Event.fPending)
3712	{
3713	SVMEVENT Event;
3714	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3715	Assert(Event.n.u1Valid);
3716
3717	/*
3718	* Validate event injection pre-conditions.
3719	*/
3720	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3721	{
3722	Assert(fAllowInt);
3723	Assert(!fIntShadow);
3724	}
3725	else if (Event.n.u3Type == SVM_EVENT_NMI)
3726	{
3727	Assert(fGif);
3728	Assert(!fIntShadow);
3729	}
3730
3731	/*
3732	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3733	* do this only when we are surely going to inject the NMI as otherwise if we return
3734	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3735	* SVM R0.
3736	*
3737	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3738	* which will set the VMCS field after actually delivering the NMI which we read on
3739	* VM-exit to determine the state.
3740	*/
3741	if ( Event.n.u3Type == SVM_EVENT_NMI
3742	&& Event.n.u8Vector == X86_XCPT_NMI)
3743	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3744
3745	/*
3746	* Inject it (update VMCB for injection by the hardware).
3747	*/
3748	Log4(("Injecting pending HM event\n"));
3749	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3750	pVCpu->hm.s.Event.fPending = false;
3751
3752	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3753	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3754	else
3755	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3756	}
3757	else
3758	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3759
3760	/*
3761	* We could have injected an NMI through IEM and continue guest execution using
3762	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3763	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3764	*/
3765	if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
3766	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3767
3768	/*
3769	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3770	*
3771	* For nested-guests: We need to update it too for the scenario where IEM executes
3772	* the nested-guest but execution later continues here with an interrupt shadow active.
3773	*/
3774	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3775	}
3776
3777
3778	/**
3779	* Reports world-switch error and dumps some useful debug info.
3780	*
3781	* @param pVCpu The cross context virtual CPU structure.
3782	* @param rcVMRun The return code from VMRUN (or
3783	* VERR_SVM_INVALID_GUEST_STATE for invalid
3784	* guest-state).
3785	*/
3786	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3787	{
3788	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3789	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3790	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3791
3792	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3793	{
3794	#ifdef VBOX_STRICT
3795	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3796	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3797	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3798	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3799	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3800	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3801	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3802	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3803	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3804	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3805	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3806	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3807
3808	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3809	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3810	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3811
3812	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3813	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3814	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3815	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3816	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3817	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3818	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3819	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3820	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3821	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3822	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3823	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3824
3825	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3826	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3827	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3828	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3829	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3830	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3831	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3832	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3833	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3834	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3835	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3836	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3837	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3838	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3839	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3840	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3841	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3842	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3843	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3844	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3845
3846	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3847
3848	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3849	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3850
3851	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3852	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3853	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3854	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3855	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3856	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3857	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3858	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3859	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3860	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3861	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3862	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3863	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3864	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3865	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3866	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3867	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3868	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3869	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3870	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3871
3872	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3873	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3874
3875	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3876	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3877	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3878	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3879
3880	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3881	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3882
3883	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3884	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3885	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3886	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3887
3888	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3889	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3890	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3891	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3892	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3893	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3894	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3895
3896	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3897	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3898	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3899	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3900
3901	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3902	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3903	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3904
3905	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3906	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3907	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3908	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3909	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3910	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3911	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3912	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3913	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3914	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3915	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3916	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3917
3918	NOREF(pVmcb);
3919	#endif /* VBOX_STRICT */
3920	}
3921	else
3922	Log4Func(("rcVMRun=%d\n", rcVMRun));
3923	}
3924
3925
3926	/**
3927	* Check per-VM and per-VCPU force flag actions that require us to go back to
3928	* ring-3 for one reason or another.
3929	*
3930	* @returns Strict VBox status code (information status code included).
3931	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3932	* ring-3.
3933	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3934	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3935	* interrupts)
3936	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3937	* all EMTs to be in ring-3.
3938	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3939	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3940	* to the EM loop.
3941	*
3942	* @param pVCpu The cross context virtual CPU structure.
3943	*/
3944	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3945	{
3946	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3947
3948	/* Could happen as a result of longjump. */
3949	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3950	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3951
3952	/* Update pending interrupts into the APIC's IRR. */
3953	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3954	APICUpdatePendingInterrupts(pVCpu);
3955
3956	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3957	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3958	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3959	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3960	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3961	{
3962	/* Pending PGM C3 sync. */
3963	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3964	{
3965	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3966	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3967	if (rc != VINF_SUCCESS)
3968	{
3969	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3970	return rc;
3971	}
3972	}
3973
3974	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3975	/* -XXX- what was that about single stepping? */
3976	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3977	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3978	{
3979	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3980	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3981	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3982	return rc;
3983	}
3984
3985	/* Pending VM request packets, such as hardware interrupts. */
3986	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3987	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3988	{
3989	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
3990	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3991	return VINF_EM_PENDING_REQUEST;
3992	}
3993
3994	/* Pending PGM pool flushes. */
3995	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3996	{
3997	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
3998	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
3999	return VINF_PGM_POOL_FLUSH_PENDING;
4000	}
4001
4002	/* Pending DMA requests. */
4003	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4004	{
4005	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4006	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4007	return VINF_EM_RAW_TO_R3;
4008	}
4009	}
4010
4011	return VINF_SUCCESS;
4012	}
4013
4014
4015	/**
4016	* Does the preparations before executing guest code in AMD-V.
4017	*
4018	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4019	* recompiler. We must be cautious what we do here regarding committing
4020	* guest-state information into the VMCB assuming we assuredly execute the guest
4021	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4022	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4023	* that the recompiler can (and should) use them when it resumes guest
4024	* execution. Otherwise such operations must be done when we can no longer
4025	* exit to ring-3.
4026	*
4027	* @returns Strict VBox status code (informational status codes included).
4028	* @retval VINF_SUCCESS if we can proceed with running the guest.
4029	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4030	*
4031	* @param pVCpu The cross context virtual CPU structure.
4032	* @param pSvmTransient Pointer to the SVM transient structure.
4033	*/
4034	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4035	{
4036	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4037
4038	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4039	if (pSvmTransient->fIsNestedGuest)
4040	{
4041	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4042	return VINF_EM_RESCHEDULE_REM;
4043	}
4044	#endif
4045
4046	/* Check force flag actions that might require us to go back to ring-3. */
4047	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4048	if (rc != VINF_SUCCESS)
4049	return rc;
4050
4051	if (TRPMHasTrap(pVCpu))
4052	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4053	else if (!pVCpu->hm.s.Event.fPending)
4054	{
4055	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4056	if ( rc != VINF_SUCCESS
4057	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4058	{
4059	/* If a nested-guest VM-exit occurred, bail. */
4060	if (pSvmTransient->fIsNestedGuest)
4061	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4062	return rc;
4063	}
4064	}
4065
4066	/*
4067	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4068	* Just do it in software, see @bugref{8411}.
4069	* NB: If we could continue a task switch exit we wouldn't need to do this.
4070	*/
4071	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4072	if (RT_UNLIKELY( !g_fHmSvmFeatures
4073	&& pVCpu->hm.s.Event.fPending
4074	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4075	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4076
4077	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4078	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4079	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4080	#endif
4081
4082	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4083	/*
4084	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4085	*/
4086	if (pSvmTransient->fIsNestedGuest)
4087	hmR0SvmSetupVmcbNested(pVCpu);
4088	#endif
4089
4090	/*
4091	* Export the guest state bits that are not shared with the host in any way as we can
4092	* longjmp or get preempted in the midst of exporting some of the state.
4093	*/
4094	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4095	AssertRCReturn(rc, rc);
4096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4097
4098	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4099	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4100
4101	/*
4102	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4103	* world-switch so we can update it on the way back if the guest changed the TPR.
4104	*/
4105	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4106	{
4107	Assert(!pSvmTransient->fIsNestedGuest);
4108	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4109	if (pVM->hm.s.fTprPatchingActive)
4110	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4111	else
4112	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4113	}
4114
4115	/*
4116	* No longjmps to ring-3 from this point on!!!
4117	*
4118	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4119	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4120	*/
4121	VMMRZCallRing3Disable(pVCpu);
4122
4123	/*
4124	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4125	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4126	* preemption disabled for a while. Since this is purly to aid the
4127	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4128	* disable interrupt on NT.
4129	*
4130	* We need to check for force-flags that could've possible been altered since we last
4131	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4132	* see @bugref{6398}).
4133	*
4134	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4135	* to ring-3 before executing guest code.
4136	*/
4137	pSvmTransient->fEFlags = ASMIntDisableFlags();
4138	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4139	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4140	{
4141	ASMSetFlags(pSvmTransient->fEFlags);
4142	VMMRZCallRing3Enable(pVCpu);
4143	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4144	return VINF_EM_RAW_TO_R3;
4145	}
4146	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4147	{
4148	ASMSetFlags(pSvmTransient->fEFlags);
4149	VMMRZCallRing3Enable(pVCpu);
4150	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4151	return VINF_EM_RAW_INTERRUPT;
4152	}
4153
4154	return VINF_SUCCESS;
4155	}
4156
4157
4158	/**
4159	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4160	* doing so.
4161	*
4162	* This means there is no backing out to ring-3 or anywhere else at this point.
4163	*
4164	* @param pVCpu The cross context virtual CPU structure.
4165	* @param pSvmTransient Pointer to the SVM transient structure.
4166	*
4167	* @remarks Called with preemption disabled.
4168	* @remarks No-long-jump zone!!!
4169	*/
4170	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4171	{
4172	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4173	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4174
4175	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4176	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4177
4178	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4179	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4180
4181	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4182
4183	if (!CPUMIsGuestFPUStateActive(pVCpu))
4184	{
4185	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4186	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4187	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4188	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4189	}
4190
4191	/* Load the state shared between host and guest (FPU, debug). */
4192	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4193	hmR0SvmExportSharedState(pVCpu, pVmcb);
4194
4195	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4196	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4197
4198	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4199	RTCPUID const idHostCpu = pHostCpu->idCpu;
4200	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4201
4202	/* Setup TSC offsetting. */
4203	if ( pSvmTransient->fUpdateTscOffsetting
4204	\|\| fMigratedHostCpu)
4205	{
4206	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4207	pSvmTransient->fUpdateTscOffsetting = false;
4208	}
4209
4210	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4211	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4212	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4213	else
4214	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4215
4216	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4217	if (fMigratedHostCpu)
4218	pVmcb->ctrl.u32VmcbCleanBits = 0;
4219
4220	/* Store status of the shared guest-host state at the time of VMRUN. */
4221	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4222	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4223
4224	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4225	uint8_t *pbMsrBitmap;
4226	if (!pSvmTransient->fIsNestedGuest)
4227	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4228	else
4229	{
4230	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4231	* MSRPM and only perform this if it changed also use EVEX.POR when it
4232	* does. */
4233	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4234
4235	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4236	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4237	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4238	}
4239	#else
4240	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4241	#endif
4242
4243	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4244	/* Flush the appropriate tagged-TLB entries. */
4245	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4246	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4247
4248	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4249
4250	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4251	to start executing. */
4252
4253	/*
4254	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4255	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4256	*
4257	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4258	*/
4259	if ( g_CpumHostFeatures.s.fRdTscP
4260	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4261	{
4262	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4263	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4264	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4265	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4266	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4267	pSvmTransient->fRestoreTscAuxMsr = true;
4268	}
4269	else
4270	{
4271	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4272	pSvmTransient->fRestoreTscAuxMsr = false;
4273	}
4274	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4275
4276	/*
4277	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4278	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4279	* from the VMCB.
4280	*/
4281	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4282	if (!fSupportsVmcbCleanBits)
4283	pVmcb->ctrl.u32VmcbCleanBits = 0;
4284	}
4285
4286
4287	/**
4288	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4289	*
4290	* @returns VBox strict status code.
4291	* @param pVCpu The cross context virtual CPU structure.
4292	* @param HCPhysVmcb The host physical address of the VMCB.
4293	*
4294	* @remarks No-long-jump zone!!!
4295	*/
4296	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4297	{
4298	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4299	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4300	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4301	}
4302
4303
4304	/**
4305	* Performs some essential restoration of state after running guest (or
4306	* nested-guest) code in AMD-V.
4307	*
4308	* @param pVCpu The cross context virtual CPU structure.
4309	* @param pSvmTransient Pointer to the SVM transient structure.
4310	* @param rcVMRun Return code of VMRUN.
4311	*
4312	* @remarks Called with interrupts disabled.
4313	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4314	* unconditionally when it is safe to do so.
4315	*/
4316	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4317	{
4318	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4319
4320	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4321	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4322
4323	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4324	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4325
4326	/* TSC read must be done early for maximum accuracy. */
4327	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4328	{
4329	if (!pSvmTransient->fIsNestedGuest)
4330	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4331	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4332	else
4333	{
4334	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4335	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4336	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4337	}
4338	#endif
4339	}
4340
4341	if (pSvmTransient->fRestoreTscAuxMsr)
4342	{
4343	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4344	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4345	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4346	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4347	}
4348
4349	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4350	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4351	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4352	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4353
4354	Assert(!(ASMGetFlags() & X86_EFL_IF));
4355	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4356	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4357
4358	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4359	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4360	{
4361	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4362	return;
4363	}
4364
4365	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4366	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4367	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4368	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4369
4370	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4371	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4372	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4373	#else
4374	/*
4375	* Always import the following:
4376	*
4377	* - RIP for exit optimizations and evaluating event injection on re-entry.
4378	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4379	* state on preemption.
4380	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4381	* - CS for exit optimizations.
4382	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4383	* assembly switcher code.
4384	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4385	*/
4386	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4387	\| CPUMCTX_EXTRN_RFLAGS
4388	\| CPUMCTX_EXTRN_RAX
4389	\| CPUMCTX_EXTRN_RSP
4390	\| CPUMCTX_EXTRN_CS
4391	\| CPUMCTX_EXTRN_HWVIRT
4392	\| CPUMCTX_EXTRN_INHIBIT_INT
4393	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4394	\| HMSVM_CPUMCTX_SHARED_STATE);
4395	#endif
4396
4397	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4398	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4399	{
4400	Assert(!pSvmTransient->fIsNestedGuest);
4401	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4402	if ( pVM->hm.s.fTprPatchingActive
4403	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4404	{
4405	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4406	AssertRC(rc);
4407	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4408	}
4409	/* Sync TPR when we aren't intercepting CR8 writes. */
4410	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4411	{
4412	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4413	AssertRC(rc);
4414	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4415	}
4416	}
4417
4418	#ifdef DEBUG_ramshankar
4419	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4420	{
4421	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4422	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4423	0 /* uVerbose */);
4424	}
4425	#endif
4426
4427	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4428	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4429	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4430	}
4431
4432
4433	/**
4434	* Runs the guest code using AMD-V.
4435	*
4436	* @returns Strict VBox status code.
4437	* @param pVCpu The cross context virtual CPU structure.
4438	* @param pcLoops Pointer to the number of executed loops.
4439	*/
4440	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4441	{
4442	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4443	Assert(pcLoops);
4444	Assert(*pcLoops <= cMaxResumeLoops);
4445
4446	SVMTRANSIENT SvmTransient;
4447	RT_ZERO(SvmTransient);
4448	SvmTransient.fUpdateTscOffsetting = true;
4449	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4450
4451	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4452	for (;;)
4453	{
4454	Assert(!HMR0SuspendPending());
4455	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4456
4457	/* Preparatory work for running nested-guest code, this may force us to return to
4458	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4459	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4460	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4461	if (rc != VINF_SUCCESS)
4462	break;
4463
4464	/*
4465	* No longjmps to ring-3 from this point on!!!
4466	*
4467	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4468	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4469	*/
4470	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4471	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4472
4473	/* Restore any residual host-state and save any bits shared between host and guest
4474	into the guest-CPU state. Re-enables interrupts! */
4475	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4476
4477	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4478	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4479	{
4480	if (rc == VINF_SUCCESS)
4481	rc = VERR_SVM_INVALID_GUEST_STATE;
4482	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4483	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4484	break;
4485	}
4486
4487	/* Handle the #VMEXIT. */
4488	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4489	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4490	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4491	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4492	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4493	if (rc != VINF_SUCCESS)
4494	break;
4495	if (++(*pcLoops) >= cMaxResumeLoops)
4496	{
4497	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4498	rc = VINF_EM_RAW_INTERRUPT;
4499	break;
4500	}
4501	}
4502
4503	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4504	return rc;
4505	}
4506
4507
4508	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4509	/**
4510	* Runs the nested-guest code using AMD-V.
4511	*
4512	* @returns Strict VBox status code.
4513	* @param pVCpu The cross context virtual CPU structure.
4514	* @param pcLoops Pointer to the number of executed loops. If we're switching
4515	* from the guest-code execution loop to this nested-guest
4516	* execution loop pass the remainder value, else pass 0.
4517	*/
4518	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4519	{
4520	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4521	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4522	Assert(pcLoops);
4523	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4524	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4525	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4526
4527	SVMTRANSIENT SvmTransient;
4528	RT_ZERO(SvmTransient);
4529	SvmTransient.fUpdateTscOffsetting = true;
4530	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4531	SvmTransient.fIsNestedGuest = true;
4532
4533	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4534	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4535
4536	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4537	for (;;)
4538	{
4539	Assert(!HMR0SuspendPending());
4540	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4541
4542	/* Preparatory work for running nested-guest code, this may force us to return to
4543	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4544	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4545	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4546	if ( rc != VINF_SUCCESS
4547	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4548	break;
4549
4550	/*
4551	* No longjmps to ring-3 from this point on!!!
4552	*
4553	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4554	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4555	*/
4556	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4557
4558	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4559
4560	/* Restore any residual host-state and save any bits shared between host and guest
4561	into the guest-CPU state. Re-enables interrupts! */
4562	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4563
4564	if (RT_LIKELY( rc == VINF_SUCCESS
4565	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4566	{ /* extremely likely */ }
4567	else
4568	{
4569	/* VMRUN failed, shouldn't really happen, Guru. */
4570	if (rc != VINF_SUCCESS)
4571	break;
4572
4573	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4574	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4575	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4576	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4577	break;
4578	}
4579
4580	/* Handle the #VMEXIT. */
4581	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4582	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4583	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4584	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4585	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4586	if (rc == VINF_SUCCESS)
4587	{
4588	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4589	{
4590	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4591	rc = VINF_SVM_VMEXIT;
4592	}
4593	else
4594	{
4595	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4596	continue;
4597	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4598	rc = VINF_EM_RAW_INTERRUPT;
4599	}
4600	}
4601	else
4602	Assert(rc != VINF_SVM_VMEXIT);
4603	break;
4604	/** @todo NSTSVM: handle single-stepping. */
4605	}
4606
4607	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4608	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4609
4610	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4611	return rc;
4612	}
4613	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4614
4615
4616	/**
4617	* Checks if any expensive dtrace probes are enabled and we should go to the
4618	* debug loop.
4619	*
4620	* @returns true if we should use debug loop, false if not.
4621	*/
4622	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4623	{
4624	/* It's probably faster to OR the raw 32-bit counter variables together.
4625	Since the variables are in an array and the probes are next to one
4626	another (more or less), we have good locality. So, better read
4627	eight-nine cache lines ever time and only have one conditional, than
4628	128+ conditionals, right? */
4629	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4630	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4631	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4632	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4633	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4634	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4635	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4636	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4637	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4638	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4639	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4644	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4645	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4648	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4649	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4650	) != 0
4651	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4652	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4653	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4654	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4655	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4656	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4657	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4658	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4659	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4660	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4677	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4679	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4680	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4681	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4682	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4683	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4684	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4685	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4686	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4687	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4689	) != 0
4690	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4691	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4692	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4693	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4694	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4695	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4696	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4697	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4698	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4699	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4717	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4719	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4720	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4721	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4722	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4723	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4724	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4725	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4726	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4729	) != 0;
4730	}
4731
4732
4733	/**
4734	* Runs the guest code using AMD-V.
4735	*
4736	* @returns Strict VBox status code.
4737	* @param pVCpu The cross context virtual CPU structure.
4738	*/
4739	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4740	{
4741	AssertPtr(pVCpu);
4742	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4743	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4744	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4745	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4746
4747	uint32_t cLoops = 0;
4748	VBOXSTRICTRC rc;
4749	for (;;)
4750	{
4751	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4752	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4753	#else
4754	NOREF(pCtx);
4755	bool const fInNestedGuestMode = false;
4756	#endif
4757	if (!fInNestedGuestMode)
4758	{
4759	if ( !pVCpu->hm.s.fUseDebugLoop
4760	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4761	&& !DBGFIsStepping(pVCpu)
4762	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4763	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4764	else
4765	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4766	}
4767	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4768	else
4769	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4770
4771	if (rc == VINF_SVM_VMRUN)
4772	{
4773	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4774	continue;
4775	}
4776	if (rc == VINF_SVM_VMEXIT)
4777	{
4778	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4779	continue;
4780	}
4781	#endif
4782	break;
4783	}
4784
4785	/* Fixup error codes. */
4786	if (rc == VERR_EM_INTERPRETER)
4787	rc = VINF_EM_RAW_EMULATE_INSTR;
4788	else if (rc == VINF_EM_RESET)
4789	rc = VINF_EM_TRIPLE_FAULT;
4790
4791	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4792	rc = hmR0SvmExitToRing3(pVCpu, rc);
4793	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4794	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4795	return rc;
4796	}
4797
4798	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4799
4800	/**
4801	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4802	*
4803	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4804	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4805	*/
4806	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4807	{
4808	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4809	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4810	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4811	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4812	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4813	const bool fRep = pIoExitInfo->n.u1Rep;
4814	const bool fStrIo = pIoExitInfo->n.u1Str;
4815
4816	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4817	NULL /* pIoExitInfo */);
4818	}
4819
4820
4821	/**
4822	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4823	* SVM_EXIT_INVALID).
4824	*
4825	* @returns VBox status code (informational status codes included).
4826	* @param pVCpu The cross context virtual CPU structure.
4827	* @param pSvmTransient Pointer to the SVM transient structure.
4828	*/
4829	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4830	{
4831	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4832	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4833	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4834
4835	/*
4836	* We import the complete state here because we use separate VMCBs for the guest and the
4837	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4838	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4839	*/
4840	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4841	do { \
4842	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4843	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4844	} while (0)
4845
4846	/*
4847	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4848	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4849	*/
4850	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4851	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4852	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4853	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4854	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4855	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4856
4857	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4858	switch (uExitCode)
4859	{
4860	case SVM_EXIT_CPUID:
4861	{
4862	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4863	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4864	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4865	}
4866
4867	case SVM_EXIT_RDTSC:
4868	{
4869	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4870	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4871	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4872	}
4873
4874	case SVM_EXIT_RDTSCP:
4875	{
4876	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4877	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4878	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4879	}
4880
4881	case SVM_EXIT_MONITOR:
4882	{
4883	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4884	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4885	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4886	}
4887
4888	case SVM_EXIT_MWAIT:
4889	{
4890	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4891	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4892	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4893	}
4894
4895	case SVM_EXIT_HLT:
4896	{
4897	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4898	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4899	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4900	}
4901
4902	case SVM_EXIT_MSR:
4903	{
4904	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4905	{
4906	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4907	uint16_t offMsrpm;
4908	uint8_t uMsrpmBit;
4909	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4910	if (RT_SUCCESS(rc))
4911	{
4912	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4913	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4914
4915	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4916	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4917	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4918
4919	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4920	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4921	{
4922	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4923	}
4924	}
4925	else
4926	{
4927	/*
4928	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4929	* See AMD-V spec. "15.11 MSR Intercepts".
4930	*/
4931	Assert(rc == VERR_OUT_OF_RANGE);
4932	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4933	}
4934	}
4935	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4936	}
4937
4938	case SVM_EXIT_IOIO:
4939	{
4940	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4941	{
4942	SVMIOIOEXITINFO IoExitInfo;
4943	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4944	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4945	if (fIntercept)
4946	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4947	}
4948	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4949	}
4950
4951	case SVM_EXIT_XCPT_PF:
4952	{
4953	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4954	if (pVM->hmr0.s.fNestedPaging)
4955	{
4956	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4957	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4958
4959	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4960	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4961	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4962
4963	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4964	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4965	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4966	return VINF_SUCCESS;
4967	}
4968	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4969	}
4970
4971	case SVM_EXIT_XCPT_UD:
4972	{
4973	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4974	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4975	hmR0SvmSetPendingXcptUD(pVCpu);
4976	return VINF_SUCCESS;
4977	}
4978
4979	case SVM_EXIT_XCPT_MF:
4980	{
4981	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4982	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4983	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4984	}
4985
4986	case SVM_EXIT_XCPT_DB:
4987	{
4988	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4989	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4990	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
4991	}
4992
4993	case SVM_EXIT_XCPT_AC:
4994	{
4995	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
4996	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4997	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
4998	}
4999
5000	case SVM_EXIT_XCPT_BP:
5001	{
5002	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5003	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5004	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5005	}
5006
5007	case SVM_EXIT_READ_CR0:
5008	case SVM_EXIT_READ_CR3:
5009	case SVM_EXIT_READ_CR4:
5010	{
5011	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5012	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5013	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5014	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5015	}
5016
5017	case SVM_EXIT_CR0_SEL_WRITE:
5018	{
5019	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5020	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5021	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5022	}
5023
5024	case SVM_EXIT_WRITE_CR0:
5025	case SVM_EXIT_WRITE_CR3:
5026	case SVM_EXIT_WRITE_CR4:
5027	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5028	{
5029	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5030	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5031
5032	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5033	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5034	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5035	}
5036
5037	case SVM_EXIT_PAUSE:
5038	{
5039	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5040	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5041	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5042	}
5043
5044	case SVM_EXIT_VINTR:
5045	{
5046	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5047	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5048	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5049	}
5050
5051	case SVM_EXIT_INTR:
5052	case SVM_EXIT_NMI:
5053	case SVM_EXIT_SMI:
5054	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5055	{
5056	/*
5057	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5058	*
5059	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5060	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5061	* nested-guest #VMEXIT.
5062	*
5063	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5064	* while trying to inject interrupts, see comment at the top of this function.
5065	*/
5066	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5067	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5068	}
5069
5070	case SVM_EXIT_FERR_FREEZE:
5071	{
5072	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5073	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5074	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5075	}
5076
5077	case SVM_EXIT_INVLPG:
5078	{
5079	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5080	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5081	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5082	}
5083
5084	case SVM_EXIT_WBINVD:
5085	{
5086	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5087	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5088	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5089	}
5090
5091	case SVM_EXIT_INVD:
5092	{
5093	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5094	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5095	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5096	}
5097
5098	case SVM_EXIT_RDPMC:
5099	{
5100	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5101	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5102	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5103	}
5104
5105	default:
5106	{
5107	switch (uExitCode)
5108	{
5109	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5110	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5111	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5112	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5113	{
5114	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5115	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5116	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5117	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5118	}
5119
5120	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5121	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5122	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5123	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5124	{
5125	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5126	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5127	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5128	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5129	}
5130
5131	case SVM_EXIT_XCPT_DE:
5132	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5133	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5134	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5135	case SVM_EXIT_XCPT_OF:
5136	case SVM_EXIT_XCPT_BR:
5137	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5138	case SVM_EXIT_XCPT_NM:
5139	case SVM_EXIT_XCPT_DF:
5140	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5141	case SVM_EXIT_XCPT_TS:
5142	case SVM_EXIT_XCPT_NP:
5143	case SVM_EXIT_XCPT_SS:
5144	case SVM_EXIT_XCPT_GP:
5145	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5146	case SVM_EXIT_XCPT_15: /* Reserved. */
5147	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5148	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5149	case SVM_EXIT_XCPT_MC:
5150	case SVM_EXIT_XCPT_XF:
5151	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5152	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5153	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5154	{
5155	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5156	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5157	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5158	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5159	}
5160
5161	case SVM_EXIT_XSETBV:
5162	{
5163	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5164	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5165	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5166	}
5167
5168	case SVM_EXIT_TASK_SWITCH:
5169	{
5170	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5171	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5172	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5173	}
5174
5175	case SVM_EXIT_IRET:
5176	{
5177	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5178	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5179	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5180	}
5181
5182	case SVM_EXIT_SHUTDOWN:
5183	{
5184	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5185	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5186	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5187	}
5188
5189	case SVM_EXIT_VMMCALL:
5190	{
5191	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5192	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5193	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5194	}
5195
5196	case SVM_EXIT_CLGI:
5197	{
5198	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5199	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5200	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5201	}
5202
5203	case SVM_EXIT_STGI:
5204	{
5205	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5206	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5207	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5208	}
5209
5210	case SVM_EXIT_VMLOAD:
5211	{
5212	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5213	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5214	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5215	}
5216
5217	case SVM_EXIT_VMSAVE:
5218	{
5219	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5220	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5221	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5222	}
5223
5224	case SVM_EXIT_INVLPGA:
5225	{
5226	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5227	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5228	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5229	}
5230
5231	case SVM_EXIT_VMRUN:
5232	{
5233	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5234	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5235	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5236	}
5237
5238	case SVM_EXIT_RSM:
5239	{
5240	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5241	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5242	hmR0SvmSetPendingXcptUD(pVCpu);
5243	return VINF_SUCCESS;
5244	}
5245
5246	case SVM_EXIT_SKINIT:
5247	{
5248	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5249	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5250	hmR0SvmSetPendingXcptUD(pVCpu);
5251	return VINF_SUCCESS;
5252	}
5253
5254	case SVM_EXIT_NPF:
5255	{
5256	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5257	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5258	}
5259
5260	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5261	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5262
5263	default:
5264	{
5265	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5266	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5267	return VERR_SVM_UNKNOWN_EXIT;
5268	}
5269	}
5270	}
5271	}
5272	/* not reached */
5273
5274	# undef NST_GST_VMEXIT_CALL_RET
5275	}
5276
5277	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5278
5279	/** @def VMEXIT_CALL_RET
5280	* Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5281	*/
5282	#ifdef DEBUG_ramshankar
5283	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5284	do { \
5285	if ((a_fDbg) == 1) \
5286	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5287	int rc = a_CallExpr; \
5288	if ((a_fDbg) == 1) \
5289	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5290	return rc; \
5291	} while (0)
5292	#else
5293	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5294	#endif
5295
5296	/**
5297	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5298	*
5299	* @returns Strict VBox status code (informational status codes included).
5300	* @param pVCpu The cross context virtual CPU structure.
5301	* @param pSvmTransient Pointer to the SVM transient structure.
5302	*/
5303	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5304	{
5305	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5306	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5307
5308	/*
5309	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5310	* for most guests under normal workloads (for some definition of "normal").
5311	*/
5312	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5313	switch (uExitCode)
5314	{
5315	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5316	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5317	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5318	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5319	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5320	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5321	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5322	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5323	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5324	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5325
5326	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5327	case SVM_EXIT_INTR:
5328	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5329
5330	case SVM_EXIT_READ_CR0:
5331	case SVM_EXIT_READ_CR3:
5332	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5333
5334	case SVM_EXIT_CR0_SEL_WRITE:
5335	case SVM_EXIT_WRITE_CR0:
5336	case SVM_EXIT_WRITE_CR3:
5337	case SVM_EXIT_WRITE_CR4:
5338	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5339
5340	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5341	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5342	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5343	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5344	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5345	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5346	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5347	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5348	case SVM_EXIT_XCPT_DE: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDE(pVCpu, pSvmTransient));
5349	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5350	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5351	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5352	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5353	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5354	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5355	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5356	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5357
5358	default:
5359	{
5360	switch (pSvmTransient->u64ExitCode)
5361	{
5362	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5363	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5364	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5365	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5366	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5367
5368	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5369	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5370	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5371	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5372	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5373
5374	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5375	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5376
5377	case SVM_EXIT_SMI:
5378	case SVM_EXIT_INIT:
5379	{
5380	/*
5381	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5382	* If it ever does, we want to know about it so log the exit code and bail.
5383	*/
5384	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5385	}
5386
5387	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5388	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5389	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5390	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5391	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5392	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5393	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5394	#else
5395	case SVM_EXIT_CLGI:
5396	case SVM_EXIT_STGI:
5397	case SVM_EXIT_VMLOAD:
5398	case SVM_EXIT_VMSAVE:
5399	case SVM_EXIT_INVLPGA:
5400	case SVM_EXIT_VMRUN:
5401	#endif
5402	case SVM_EXIT_RSM:
5403	case SVM_EXIT_SKINIT:
5404	{
5405	hmR0SvmSetPendingXcptUD(pVCpu);
5406	return VINF_SUCCESS;
5407	}
5408
5409	/*
5410	* The remaining should only be possible when debugging or dtracing.
5411	*/
5412	case SVM_EXIT_XCPT_DE:
5413	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5414	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5415	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5416	case SVM_EXIT_XCPT_OF:
5417	case SVM_EXIT_XCPT_BR:
5418	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5419	case SVM_EXIT_XCPT_NM:
5420	case SVM_EXIT_XCPT_DF:
5421	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5422	case SVM_EXIT_XCPT_TS:
5423	case SVM_EXIT_XCPT_NP:
5424	case SVM_EXIT_XCPT_SS:
5425	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5426	/* SVM_EXIT_XCPT_PF: */
5427	case SVM_EXIT_XCPT_15: /* Reserved. */
5428	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5429	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5430	case SVM_EXIT_XCPT_MC:
5431	case SVM_EXIT_XCPT_XF:
5432	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5433	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5434	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5435	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5436
5437	case SVM_EXIT_SWINT: VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5438	case SVM_EXIT_TR_READ: VMEXIT_CALL_RET(0, hmR0SvmExitTrRead(pVCpu, pSvmTransient));
5439	case SVM_EXIT_TR_WRITE: VMEXIT_CALL_RET(0, hmR0SvmExitTrWrite(pVCpu, pSvmTransient)); /* Also OS/2 TLB workaround. */
5440
5441	default:
5442	{
5443	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5444	pVCpu->hm.s.u32HMError = uExitCode;
5445	return VERR_SVM_UNKNOWN_EXIT;
5446	}
5447	}
5448	}
5449	}
5450	/* not reached */
5451	}
5452
5453
5454	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5455	*
5456	* The following few functions and associated structure contains the bloat
5457	* necessary for providing detailed debug events and dtrace probes as well as
5458	* reliable host side single stepping. This works on the principle of
5459	* "subclassing" the normal execution loop and workers. We replace the loop
5460	* method completely and override selected helpers to add necessary adjustments
5461	* to their core operation.
5462	*
5463	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5464	* any performance for debug and analysis features.
5465	*
5466	* @{
5467	*/
5468
5469	/**
5470	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
5471	* the debug run loop.
5472	*/
5473	typedef struct SVMRUNDBGSTATE
5474	{
5475	/** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5476	uint64_t bmInterceptInitial;
5477	/** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5478	uint32_t bmXcptInitial;
5479	/** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5480	uint16_t bmInterceptRdCRxInitial;
5481	/** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5482	uint16_t bmInterceptWrCRxInitial;
5483	/** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5484	uint16_t bmInterceptRdDRxInitial;
5485	/** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5486	uint16_t bmInterceptWrDRxInitial;
5487
5488	/** Whether we've actually modified the intercept control qword. */
5489	bool fModifiedInterceptCtrl : 1;
5490	/** Whether we've actually modified the exception bitmap. */
5491	bool fModifiedXcptBitmap : 1;
5492	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5493	bool fModifiedInterceptRdCRx : 1;
5494	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5495	bool fModifiedInterceptWrCRx : 1;
5496	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5497	bool fModifiedInterceptRdDRx : 1;
5498	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5499	bool fModifiedInterceptWrDRx : 1;
5500
5501	/** The CS we started executing with. */
5502	uint16_t uCsStart;
5503	/** The RIP we started executing at. This is for detecting that we stepped. */
5504	uint64_t uRipStart;
5505
5506	/** The sequence number of the Dtrace provider settings the state was
5507	* configured against. */
5508	uint32_t uDtraceSettingsSeqNo;
5509	/** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5510	uint32_t bmXcptExtra;
5511	/** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5512	uint64_t bmInterceptExtra;
5513	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5514	uint16_t bmInterceptRdCRxExtra;
5515	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5516	uint16_t bmInterceptWrCRxExtra;
5517	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5518	uint16_t bmInterceptRdDRxExtra;
5519	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5520	uint16_t bmInterceptWrDRxExtra;
5521	/** VM-exits to check (one bit per VM-exit). */
5522	uint32_t bmExitsToCheck[33];
5523	} SVMRUNDBGSTATE;
5524	AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5525	typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5526
5527
5528	/**
5529	* Initializes the SVMRUNDBGSTATE structure.
5530	*
5531	* @param pVCpu The cross context virtual CPU structure of the
5532	* calling EMT.
5533	* @param pSvmTransient The SVM-transient structure.
5534	* @param pDbgState The debug state to initialize.
5535	*/
5536	static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5537	{
5538	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5539	pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5540	pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5541	pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5542	pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5543	pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5544	pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5545
5546	pDbgState->fModifiedInterceptCtrl = false;
5547	pDbgState->fModifiedXcptBitmap = false;
5548	pDbgState->fModifiedInterceptRdCRx = false;
5549	pDbgState->fModifiedInterceptWrCRx = false;
5550	pDbgState->fModifiedInterceptRdDRx = false;
5551	pDbgState->fModifiedInterceptWrDRx = false;
5552
5553	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5554	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5555
5556	/* We don't really need to zero these. */
5557	pDbgState->bmInterceptExtra = 0;
5558	pDbgState->bmXcptExtra = 0;
5559	pDbgState->bmInterceptRdCRxExtra = 0;
5560	pDbgState->bmInterceptWrCRxExtra = 0;
5561	pDbgState->bmInterceptRdDRxExtra = 0;
5562	pDbgState->bmInterceptWrDRxExtra = 0;
5563	}
5564
5565
5566	/**
5567	* Updates the VMCB fields with changes requested by @a pDbgState.
5568	*
5569	* This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5570	* immediately before executing guest code, i.e. when interrupts are disabled.
5571	* We don't check status codes here as we cannot easily assert or return in the
5572	* latter case.
5573	*
5574	* @param pSvmTransient The SVM-transient structure.
5575	* @param pDbgState The debug state.
5576	*/
5577	static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5578	{
5579	/*
5580	* Ensure desired flags in VMCS control fields are set.
5581	*/
5582	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5583	#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5584	if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5585	{ \
5586	pVmcb->ctrl. a_VmcbCtrlField \|= (a_bmExtra); \
5587	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5588	Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5589	(a_fModified) = true; \
5590	} \
5591	} while (0)
5592	ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5593	ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5594	ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5595	ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5596	ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5597	ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5598	#undef ADD_EXTRA_INTERCEPTS
5599	}
5600
5601
5602	/**
5603	* Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5604	* for re-entry next time around.
5605	*
5606	* @param pSvmTransient The SVM-transient structure.
5607	* @param pDbgState The debug state.
5608	*/
5609	static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5610	{
5611	/*
5612	* Restore VM-exit control settings as we may not reenter this function the
5613	* next time around.
5614	*/
5615	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5616
5617	#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5618	if ((a_fModified)) \
5619	{ \
5620	pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5621	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5622	} \
5623	} while (0)
5624	RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5625	RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5626	RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5627	RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5628	RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5629	RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5630	#undef RESTORE_INTERCEPTS
5631	}
5632
5633
5634	/**
5635	* Configures VM-exit controls for current DBGF and DTrace settings.
5636	*
5637	* This updates @a pDbgState and the VMCB execution control fields (in the debug
5638	* state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5639	*
5640	* @param pVCpu The cross context virtual CPU structure.
5641	* @param pSvmTransient The SVM-transient structure. May update
5642	* fUpdatedTscOffsettingAndPreemptTimer.
5643	* @param pDbgState The debug state.
5644	*/
5645	static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5646	{
5647	/*
5648	* Take down the dtrace serial number so we can spot changes.
5649	*/
5650	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5651	ASMCompilerBarrier();
5652
5653	/*
5654	* Clear data members that we'll be rebuilding here.
5655	*/
5656	pDbgState->bmXcptExtra = 0;
5657	pDbgState->bmInterceptExtra = 0;
5658	pDbgState->bmInterceptRdCRxExtra = 0;
5659	pDbgState->bmInterceptWrCRxExtra = 0;
5660	pDbgState->bmInterceptRdDRxExtra = 0;
5661	pDbgState->bmInterceptWrDRxExtra = 0;
5662	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5663	pDbgState->bmExitsToCheck[i] = 0;
5664
5665	/*
5666	* Software interrupts (INT XXh)
5667	*/
5668	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5669	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5670	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
5671	{
5672	pDbgState->bmInterceptExtra \|= SVM_CTRL_INTERCEPT_INTN;
5673	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5674	}
5675
5676	/*
5677	* INT3 breakpoints - triggered by #BP exceptions.
5678	*/
5679	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
5680	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
5681
5682	/*
5683	* Exception bitmap and XCPT events+probes.
5684	*/
5685	#define SET_XCPT(a_iXcpt) do { \
5686	pDbgState->bmXcptExtra \|= RT_BIT_32(a_iXcpt); \
5687	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5688	} while (0)
5689
5690	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5691	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5692	SET_XCPT(iXcpt);
5693
5694	if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5695	if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5696	if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5697	if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5698	if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5699	if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5700	if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5701	if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5702	if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5703	if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5704	if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5705	if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5706	if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5707	if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5708	if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5709	if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5710	if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5711	if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5712
5713	#undef SET_XCPT
5714
5715	/*
5716	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5717	*
5718	* Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5719	* So, when adding/changing/removing please don't forget to update it.
5720	*
5721	* Some of the macros are picking up local variables to save horizontal space,
5722	* (being able to see it in a table is the lesser evil here).
5723	*/
5724	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5725	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5726	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5727	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5728	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5729	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5730	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5731	} else do { } while (0)
5732	#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5733	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5734	{ \
5735	(pDbgState)->bmInterceptExtra \|= (a_fInterceptCtrl); \
5736	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5737	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5738	} else do { } while (0)
5739
5740	/** @todo double check these */
5741	/** @todo Check what more AMD-V specific we can intercept. */
5742	//SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5743	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5744	SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5745	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5746	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5747	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5748	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5749	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5750	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5751	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5752	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5753	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5754	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5755	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5756
5757	SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5758	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5759	SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5761	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5762	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5763	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5764	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5765	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5766	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5767	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5768	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5769	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5771	SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5772	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5773
5774	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5775	pDbgState->bmInterceptRdCRxExtra = 0xffff;
5776	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5777	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5778
5779	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5780	pDbgState->bmInterceptWrCRxExtra = 0xffff;
5781	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5782	{
5783	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5784	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5785	}
5786
5787	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5788	pDbgState->bmInterceptRdDRxExtra = 0xffff;
5789	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5790	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5791
5792	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5793	pDbgState->bmInterceptWrDRxExtra = 0xffff;
5794	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5795	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5796
5797	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5798	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5799	SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5800	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5801	SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5802	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5803	if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5804	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5805	SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5806	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5807	SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5808	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5809	SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5810	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5811	SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5812	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5813	SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5814	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5815	SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5816	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5817	SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5818	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5819	SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5820	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5821	SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5822	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5823	SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5824	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5825	SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5826	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5827	SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5828	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5829
5830	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5831	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5832
5833	#undef IS_EITHER_ENABLED
5834	#undef SET_ONLY_XBM_IF_EITHER_EN
5835	#undef SET_INCP_XBM_IF_EITHER_EN
5836
5837	/*
5838	* Sanitize the control stuff.
5839	*/
5840	/** @todo filter out unsupported stuff? */
5841	if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5842	!= RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
5843	{
5844	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5845	/// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5846	RT_NOREF(pSvmTransient);
5847	}
5848
5849	Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5850	pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5851	pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5852	pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5853	pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5854	pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5855	pDbgState->bmExitsToCheck[0],
5856	pDbgState->bmExitsToCheck[1],
5857	pDbgState->bmExitsToCheck[2],
5858	pDbgState->bmExitsToCheck[3],
5859	pDbgState->bmExitsToCheck[4]));
5860	}
5861
5862
5863	/**
5864	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
5865	* appropriate.
5866	*
5867	* The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5868	* bitmap.
5869	*
5870	* @returns Strict VBox status code (i.e. informational status codes too).
5871	* @param pVCpu The cross context virtual CPU structure.
5872	* @param pSvmTransient The SVM-transient structure.
5873	* @param uExitCode The VM-exit code.
5874	*
5875	* @remarks The name of this function is displayed by dtrace, so keep it short
5876	* and to the point. No longer than 33 chars long, please.
5877	*/
5878	static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5879	{
5880	/*
5881	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5882	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
5883	*
5884	* Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5885	* does. Must add/change/remove both places. Same ordering, please.
5886	*
5887	* Added/removed events must also be reflected in the next section
5888	* where we dispatch dtrace events.
5889	*/
5890	bool fDtrace1 = false;
5891	bool fDtrace2 = false;
5892	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5893	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5894	uint64_t uEventArg = 0;
5895	#define SET_XCPT(a_XcptName) \
5896	do { \
5897	enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5898	fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5899	} while (0)
5900	#define SET_EXIT(a_EventSubName) \
5901	do { \
5902	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5903	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5904	} while (0)
5905	#define SET_BOTH(a_EventSubName) \
5906	do { \
5907	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5908	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5909	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5910	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5911	} while (0)
5912	switch (uExitCode)
5913	{
5914	case SVM_EXIT_SWINT:
5915	enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5916	fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5917	uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5918	break;
5919
5920	case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5921	case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5922	case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5923	case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5924	case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5925	case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5926	case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5927	case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5928	case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5929	case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5930	case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5931	case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5932	case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5933	case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5934	case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5935	case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5936	case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5937	case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5938
5939	case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5940	case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5941	case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5942	case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5943	case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5944	case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5945	case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5946	case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5947	case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5948	case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5949	case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5950	case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5951	case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5952	case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5953
5954	case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5955	case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5956	case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5957	case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5958	case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5959	case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5960	case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5961	case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5962	case SVM_EXIT_HLT: SET_BOTH(HALT); break;
5963	case SVM_EXIT_INVD: SET_BOTH(INVD); break;
5964	case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
5965	case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
5966	case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
5967	case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
5968	case SVM_EXIT_RSM: SET_BOTH(RSM); break;
5969
5970	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
5971	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
5972	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
5973	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
5974	SET_BOTH(CRX_READ);
5975	uEventArg = uExitCode - SVM_EXIT_READ_CR0;
5976	break;
5977	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
5978	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
5979	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
5980	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
5981	case SVM_EXIT_CR0_SEL_WRITE:
5982	SET_BOTH(CRX_WRITE);
5983	uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
5984	break;
5985	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5986	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
5987	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
5988	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5989	SET_BOTH(DRX_READ);
5990	uEventArg = uExitCode - SVM_EXIT_READ_DR0;
5991	break;
5992	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5993	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
5994	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
5995	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5996	SET_BOTH(DRX_WRITE);
5997	uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
5998	break;
5999	case SVM_EXIT_MSR:
6000	if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6001	SET_BOTH(WRMSR);
6002	else
6003	SET_BOTH(RDMSR);
6004	break;
6005	case SVM_EXIT_MWAIT_ARMED:
6006	case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6007	case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6008	case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6009	case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6010	case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6011	case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6012	case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6013	case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6014	case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6015	case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6016	case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6017	case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6018	case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6019
6020	case SVM_EXIT_SHUTDOWN:
6021	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6022	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6023	break;
6024
6025	default:
6026	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6027	break;
6028	}
6029	#undef SET_BOTH
6030	#undef SET_EXIT
6031
6032	/*
6033	* Dtrace tracepoints go first. We do them here at once so we don't
6034	* have to copy the guest state saving and stuff a few dozen times.
6035	* Down side is that we've got to repeat the switch, though this time
6036	* we use enmEvent since the probes are a subset of what DBGF does.
6037	*/
6038	if (fDtrace1 \|\| fDtrace2)
6039	{
6040	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6041	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; RT_NOREF(pCtx); /* Shut up Clang 13. */
6042	switch (enmEvent1)
6043	{
6044	/** @todo consider which extra parameters would be helpful for each probe. */
6045	case DBGFEVENT_END: break;
6046	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6047	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6048	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6049	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6050	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6051	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6052	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6053	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6054	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6055	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6056	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6057	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6058	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6059	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6060	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6061	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6062	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6063	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6064	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6065	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6066	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6067	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6068	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6069	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6070	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6071	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6072	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6073	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6074	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6075	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6076	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6077	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6078	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6079	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6080	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6081	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6082	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6083	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6084	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6085	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6086	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6087	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6088	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6089	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6090	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6091	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6092	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6093	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6094	case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6095	case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6096	case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6097	case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6098	case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6099	default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6100	}
6101	switch (enmEvent2)
6102	{
6103	/** @todo consider which extra parameters would be helpful for each probe. */
6104	case DBGFEVENT_END: break;
6105	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6106	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6107	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6108	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6109	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6110	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6111	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6112	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6113	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6114	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6115	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6116	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6117	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6118	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6119	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6120	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6121	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6122	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6123	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6124	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6125	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6126	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6127	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6128	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6129	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6130	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6131	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6132	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6133	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6134	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6135	case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6136	case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6137	case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6138	case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6139	case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6140	default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6141	}
6142	}
6143
6144	/*
6145	* Fire of the DBGF event, if enabled (our check here is just a quick one,
6146	* the DBGF call will do a full check).
6147	*
6148	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6149	* Note! If we have to events, we prioritize the first, i.e. the instruction
6150	* one, in order to avoid event nesting.
6151	*/
6152	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6153	VBOXSTRICTRC rcStrict;
6154	if ( enmEvent1 != DBGFEVENT_END
6155	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6156	{
6157	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6158	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6159	}
6160	else if ( enmEvent2 != DBGFEVENT_END
6161	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6162	{
6163	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6164	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6165	}
6166	else
6167	rcStrict = VINF_SUCCESS;
6168	return rcStrict;
6169	}
6170
6171
6172	/**
6173	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6174	* debug variant.
6175	*
6176	* @returns Strict VBox status code (informational status codes included).
6177	* @param pVCpu The cross context virtual CPU structure.
6178	* @param pSvmTransient Pointer to the SVM transient structure.
6179	* @param pDbgState The runtime debug state.
6180	*/
6181	static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6182	{
6183	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6184	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6185
6186	/*
6187	* Expensive (saves context) generic dtrace VM-exit probe.
6188	*/
6189	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6190	if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6191	{ /* more likely */ }
6192	else
6193	{
6194	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6195	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6196	}
6197
6198	/*
6199	* Check for single stepping event if we're stepping.
6200	*/
6201	if (pVCpu->hm.s.fSingleInstruction)
6202	{
6203	switch (uExitCode)
6204	{
6205	/* Various events: */
6206	case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6207	case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6208	case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6209	case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6210	case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6211	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6212	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6213	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6214	case SVM_EXIT_INTR:
6215	case SVM_EXIT_NMI:
6216	case SVM_EXIT_VINTR:
6217	case SVM_EXIT_NPF:
6218	case SVM_EXIT_AVIC_NOACCEL:
6219
6220	/* Instruction specific VM-exits: */
6221	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6222	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6223	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6224	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6225	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6226	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6227	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6228	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6229	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6230	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6231	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6232	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6233	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6234	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6235	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6236	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6237	case SVM_EXIT_CR0_SEL_WRITE:
6238	case SVM_EXIT_IDTR_READ:
6239	case SVM_EXIT_GDTR_READ:
6240	case SVM_EXIT_LDTR_READ:
6241	case SVM_EXIT_TR_READ:
6242	case SVM_EXIT_IDTR_WRITE:
6243	case SVM_EXIT_GDTR_WRITE:
6244	case SVM_EXIT_LDTR_WRITE:
6245	case SVM_EXIT_TR_WRITE:
6246	case SVM_EXIT_RDTSC:
6247	case SVM_EXIT_RDPMC:
6248	case SVM_EXIT_PUSHF:
6249	case SVM_EXIT_POPF:
6250	case SVM_EXIT_CPUID:
6251	case SVM_EXIT_RSM:
6252	case SVM_EXIT_IRET:
6253	case SVM_EXIT_SWINT:
6254	case SVM_EXIT_INVD:
6255	case SVM_EXIT_PAUSE:
6256	case SVM_EXIT_HLT:
6257	case SVM_EXIT_INVLPG:
6258	case SVM_EXIT_INVLPGA:
6259	case SVM_EXIT_IOIO:
6260	case SVM_EXIT_MSR:
6261	case SVM_EXIT_TASK_SWITCH:
6262	case SVM_EXIT_VMRUN:
6263	case SVM_EXIT_VMMCALL:
6264	case SVM_EXIT_VMLOAD:
6265	case SVM_EXIT_VMSAVE:
6266	case SVM_EXIT_STGI:
6267	case SVM_EXIT_CLGI:
6268	case SVM_EXIT_SKINIT:
6269	case SVM_EXIT_RDTSCP:
6270	case SVM_EXIT_ICEBP:
6271	case SVM_EXIT_WBINVD:
6272	case SVM_EXIT_MONITOR:
6273	case SVM_EXIT_MWAIT:
6274	case SVM_EXIT_MWAIT_ARMED:
6275	case SVM_EXIT_XSETBV:
6276	case SVM_EXIT_RDPRU:
6277	case SVM_EXIT_WRITE_EFER_TRAP:
6278	case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6279	case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6280	case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6281	case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6282	case SVM_EXIT_MCOMMIT:
6283	{
6284	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6285	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6286	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6287	{
6288	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6289	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6290	return VINF_EM_DBG_STEPPED;
6291	}
6292	break;
6293	}
6294
6295	/* Errors and unexpected events: */
6296	case SVM_EXIT_FERR_FREEZE:
6297	case SVM_EXIT_SHUTDOWN:
6298	case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6299	break;
6300
6301	case SVM_EXIT_SMI:
6302	case SVM_EXIT_INIT:
6303	default:
6304	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6305	break;
6306	}
6307	}
6308
6309	/*
6310	* Check for debugger event breakpoints and dtrace probes.
6311	*/
6312	if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6313	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6314	{
6315	VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6316	if (rcStrict != VINF_SUCCESS)
6317	{
6318	Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6319	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6320	return rcStrict;
6321	}
6322	}
6323
6324	/*
6325	* Normal processing.
6326	*/
6327	return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6328	}
6329
6330
6331	/**
6332	* Runs the guest code using AMD-V in single step mode.
6333	*
6334	* @returns Strict VBox status code.
6335	* @param pVCpu The cross context virtual CPU structure.
6336	* @param pcLoops Pointer to the number of executed loops.
6337	*/
6338	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6339	{
6340	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6341	Assert(pcLoops);
6342	Assert(*pcLoops <= cMaxResumeLoops);
6343
6344	SVMTRANSIENT SvmTransient;
6345	RT_ZERO(SvmTransient);
6346	SvmTransient.fUpdateTscOffsetting = true;
6347	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6348
6349	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6350
6351	/* Set HMCPU indicators. */
6352	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6353	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
6354	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6355	pVCpu->hmr0.s.fUsingDebugLoop = true;
6356
6357	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6358	SVMRUNDBGSTATE DbgState;
6359	hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6360	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6361
6362	/*
6363	* The loop.
6364	*/
6365	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6366	for (;;)
6367	{
6368	Assert(!HMR0SuspendPending());
6369	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6370	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6371	(unsigned)RTMpCpuId(), *pcLoops));
6372	bool fStepping = pVCpu->hm.s.fSingleInstruction;
6373
6374	/* Set up VM-execution controls the next two can respond to. */
6375	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6376
6377	/* Preparatory work for running nested-guest code, this may force us to return to
6378	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6379	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6380	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6381	if (rc != VINF_SUCCESS)
6382	break;
6383
6384	/*
6385	* No longjmps to ring-3 from this point on!!!
6386	*
6387	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6388	* better than a kernel panic. This also disables flushing of the R0-logger instance.
6389	*/
6390	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6391
6392	/* Override any obnoxious code in the above two calls. */
6393	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6394	#if 0
6395	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64\n",
6396	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6397	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u));
6398	#endif
6399
6400	/*
6401	* Finally execute guest code.
6402	*/
6403	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6404
6405	/* Restore any residual host-state and save any bits shared between host and guest
6406	into the guest-CPU state. Re-enables interrupts! */
6407	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6408	#if 0
6409	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64 exit=%d\n",
6410	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6411	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u, SvmTransient.u64ExitCode));
6412	#endif
6413
6414	if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6415	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6416	{ /* very likely*/ }
6417	else
6418	{
6419	if (rc == VINF_SUCCESS)
6420	rc = VERR_SVM_INVALID_GUEST_STATE;
6421	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6422	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6423	return rc;
6424	}
6425
6426	/* Handle the #VMEXIT. */
6427	HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6428	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6429	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6430	rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6431	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6432	if (rc != VINF_SUCCESS)
6433	break;
6434	if (++(*pcLoops) >= cMaxResumeLoops)
6435	{
6436	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6437	rc = VINF_EM_RAW_INTERRUPT;
6438	break;
6439	}
6440
6441	/*
6442	* Stepping: Did the RIP change, if so, consider it a single step.
6443	* Otherwise, make sure one of the TFs gets set.
6444	*/
6445	if (fStepping)
6446	{
6447	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6448	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6449	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6450	{
6451	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6452	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6453	rc = VINF_EM_DBG_STEPPED;
6454	break;
6455	}
6456	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6457	}
6458
6459	/*
6460	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
6461	* Revert the state changes afterware so we can drop intercepts no longer needed.
6462	*/
6463	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6464	{
6465	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6466	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6467	}
6468	}
6469
6470	/*
6471	* Clear the X86_EFL_TF if necessary.
6472	*/
6473	if (pVCpu->hmr0.s.fClearTrapFlag)
6474	{
6475	pVCpu->hmr0.s.fClearTrapFlag = false;
6476	pCtx->eflags.Bits.u1TF = 0;
6477	}
6478
6479	/* Restore HMCPU indicators. */
6480	pVCpu->hmr0.s.fUsingDebugLoop = false;
6481	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6482	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6483
6484	/* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6485	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6486
6487	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6488	return rc;
6489	}
6490
6491	/** @} */
6492
6493	#undef VMEXIT_CALL_RET
6494
6495
6496	#ifdef VBOX_STRICT
6497	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6498	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6499	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6500
6501	# define HMSVM_ASSERT_PREEMPT_CPUID() \
6502	do \
6503	{ \
6504	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6505	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6506	} while (0)
6507
6508	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6509	do { \
6510	AssertPtr((a_pVCpu)); \
6511	AssertPtr((a_pSvmTransient)); \
6512	Assert(ASMIntAreEnabled()); \
6513	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6514	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6515	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6516	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6517	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6518	HMSVM_ASSERT_PREEMPT_CPUID(); \
6519	} while (0)
6520	#else
6521	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6522	do { \
6523	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6524	} while (0)
6525	#endif
6526
6527
6528	/**
6529	* Gets the IEM exception flags for the specified SVM event.
6530	*
6531	* @returns The IEM exception flags.
6532	* @param pEvent Pointer to the SVM event.
6533	*
6534	* @remarks This function currently only constructs flags required for
6535	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6536	* and CR2 aspects of an exception are not included).
6537	*/
6538	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6539	{
6540	uint8_t const uEventType = pEvent->n.u3Type;
6541	uint32_t fIemXcptFlags;
6542	switch (uEventType)
6543	{
6544	case SVM_EVENT_EXCEPTION:
6545	/*
6546	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6547	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6548	*/
6549	if (pEvent->n.u8Vector == X86_XCPT_BP)
6550	{
6551	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6552	break;
6553	}
6554	if (pEvent->n.u8Vector == X86_XCPT_OF)
6555	{
6556	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6557	break;
6558	}
6559	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6560	RT_FALL_THRU();
6561	case SVM_EVENT_NMI:
6562	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6563	break;
6564
6565	case SVM_EVENT_EXTERNAL_IRQ:
6566	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6567	break;
6568
6569	case SVM_EVENT_SOFTWARE_INT:
6570	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6571	break;
6572
6573	default:
6574	fIemXcptFlags = 0;
6575	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6576	break;
6577	}
6578	return fIemXcptFlags;
6579	}
6580
6581
6582	/**
6583	* Handle a condition that occurred while delivering an event through the guest
6584	* IDT.
6585	*
6586	* @returns VBox status code (informational error codes included).
6587	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6588	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6589	* continue execution of the guest which will delivery the \#DF.
6590	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6591	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6592	*
6593	* @param pVCpu The cross context virtual CPU structure.
6594	* @param pSvmTransient Pointer to the SVM transient structure.
6595	*
6596	* @remarks No-long-jump zone!!!
6597	*/
6598	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6599	{
6600	/** @todo r=bird: Looks like this is called on many exits and we start by
6601	* loading CR2 on the offchance that we actually have work to do here.
6602	*
6603	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY can surely check
6604	* pVmcb->ctrl.ExitIntInfo.n.u1Valid, can't it?
6605	*
6606	* Also, what's the deal with hmR0SvmGetCurrentVmcb() vs pSvmTransient->pVmcb?
6607	*/
6608	int rc = VINF_SUCCESS;
6609	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6610	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6611
6612	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6613	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6614	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6615
6616	/*
6617	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6618	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6619	*
6620	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6621	*/
6622	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6623	{
6624	IEMXCPTRAISE enmRaise;
6625	IEMXCPTRAISEINFO fRaiseInfo;
6626	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6627	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6628	if (fExitIsHwXcpt)
6629	{
6630	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6631	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6632	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6633	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6634	}
6635	else
6636	{
6637	/*
6638	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6639	* then we end up here.
6640	*
6641	* If the event was:
6642	* - a software interrupt, we can re-execute the instruction which will
6643	* regenerate the event.
6644	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6645	* - a hardware exception or external interrupt, we re-inject it.
6646	*/
6647	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6648	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6649	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6650	else
6651	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6652	}
6653
6654	switch (enmRaise)
6655	{
6656	case IEMXCPTRAISE_CURRENT_XCPT:
6657	case IEMXCPTRAISE_PREV_EVENT:
6658	{
6659	/* For software interrupts, we shall re-execute the instruction. */
6660	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6661	{
6662	RTGCUINTPTR GCPtrFaultAddress = 0;
6663
6664	/* If we are re-injecting an NMI, clear NMI blocking. */
6665	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6666	CPUMClearInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
6667
6668	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6669	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6670	{
6671	pSvmTransient->fVectoringPF = true;
6672	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6673	pVCpu->cpum.GstCtx.cr2));
6674	}
6675	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6676	&& uIdtVector == X86_XCPT_PF)
6677	{
6678	/*
6679	* If the previous exception was a #PF, we need to recover the CR2 value.
6680	* This can't happen with shadow paging.
6681	*/
6682	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6683	}
6684
6685	/*
6686	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6687	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6688	*/
6689	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6690	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6691	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6692
6693	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6694	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6695	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6696	}
6697	break;
6698	}
6699
6700	case IEMXCPTRAISE_REEXEC_INSTR:
6701	{
6702	Assert(rc == VINF_SUCCESS);
6703	break;
6704	}
6705
6706	case IEMXCPTRAISE_DOUBLE_FAULT:
6707	{
6708	/*
6709	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6710	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6711	* converted into a #DF.
6712	*/
6713	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6714	{
6715	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6716	pSvmTransient->fVectoringDoublePF = true;
6717	Assert(rc == VINF_SUCCESS);
6718	}
6719	else
6720	{
6721	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6722	hmR0SvmSetPendingXcptDF(pVCpu);
6723	rc = VINF_HM_DOUBLE_FAULT;
6724	}
6725	break;
6726	}
6727
6728	case IEMXCPTRAISE_TRIPLE_FAULT:
6729	{
6730	rc = VINF_EM_RESET;
6731	break;
6732	}
6733
6734	case IEMXCPTRAISE_CPU_HANG:
6735	{
6736	rc = VERR_EM_GUEST_CPU_HANG;
6737	break;
6738	}
6739
6740	default:
6741	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6742	}
6743	}
6744	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6745	return rc;
6746	}
6747
6748
6749	/**
6750	* Advances the guest RIP by the number of bytes specified in @a cb.
6751	*
6752	* @param pVCpu The cross context virtual CPU structure.
6753	* @param cb RIP increment value in bytes.
6754	*/
6755	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6756	{
6757	pVCpu->cpum.GstCtx.rip += cb;
6758	CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
6759	/** @todo clear RF. */
6760	}
6761
6762
6763	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6764	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6765	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6766
6767	/** @name \#VMEXIT handlers.
6768	* @{
6769	*/
6770
6771	/**
6772	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6773	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6774	*/
6775	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6776	{
6777	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6778
6779	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6780	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6781	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6782	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6783
6784	/*
6785	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6786	* signal -before- the timer fires if the current interrupt is our own timer or a some
6787	* other host interrupt. We also cannot examine what interrupt it is until the host
6788	* actually take the interrupt.
6789	*
6790	* Going back to executing guest code here unconditionally causes random scheduling
6791	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6792	*/
6793	return VINF_EM_RAW_INTERRUPT;
6794	}
6795
6796
6797	/**
6798	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6799	*/
6800	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6801	{
6802	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6803
6804	VBOXSTRICTRC rcStrict;
6805	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6806	if (fSupportsNextRipSave)
6807	{
6808	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6809	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6810	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6811	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6812	}
6813	else
6814	{
6815	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6816	rcStrict = IEMExecOne(pVCpu);
6817	}
6818
6819	if (rcStrict == VINF_IEM_RAISED_XCPT)
6820	{
6821	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6822	rcStrict = VINF_SUCCESS;
6823	}
6824	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6825	return rcStrict;
6826	}
6827
6828
6829	/**
6830	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6831	*/
6832	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6833	{
6834	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6835
6836	VBOXSTRICTRC rcStrict;
6837	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6838	if (fSupportsNextRipSave)
6839	{
6840	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6841	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6842	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6843	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6844	}
6845	else
6846	{
6847	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6848	rcStrict = IEMExecOne(pVCpu);
6849	}
6850
6851	if (rcStrict == VINF_IEM_RAISED_XCPT)
6852	{
6853	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6854	rcStrict = VINF_SUCCESS;
6855	}
6856	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6857	return rcStrict;
6858	}
6859
6860
6861	/**
6862	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6863	*/
6864	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6865	{
6866	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6867
6868	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6869	VBOXSTRICTRC rcStrict;
6870	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6871	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6872	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6873	if (!pExitRec)
6874	{
6875	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6876	if (fSupportsNextRipSave)
6877	{
6878	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6879	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6880	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6881	}
6882	else
6883	{
6884	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6885	rcStrict = IEMExecOne(pVCpu);
6886	}
6887
6888	if (rcStrict == VINF_IEM_RAISED_XCPT)
6889	{
6890	CPUM_ASSERT_NOT_EXTRN(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
6891	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6892	rcStrict = VINF_SUCCESS;
6893	}
6894	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6895	}
6896	else
6897	{
6898	/*
6899	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6900	*/
6901	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6902
6903	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6904	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6905
6906	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6907
6908	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6909	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6910	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6911	}
6912	return rcStrict;
6913	}
6914
6915
6916	/**
6917	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6918	*/
6919	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6920	{
6921	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6922
6923	VBOXSTRICTRC rcStrict;
6924	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6925	if (fSupportsNextRipSave)
6926	{
6927	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6928	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6929	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6930	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6931	}
6932	else
6933	{
6934	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6935	rcStrict = IEMExecOne(pVCpu);
6936	}
6937
6938	if (rcStrict == VINF_SUCCESS)
6939	pSvmTransient->fUpdateTscOffsetting = true;
6940	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6941	{
6942	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6943	rcStrict = VINF_SUCCESS;
6944	}
6945	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6946	return rcStrict;
6947	}
6948
6949
6950	/**
6951	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6952	*/
6953	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6954	{
6955	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6956
6957	VBOXSTRICTRC rcStrict;
6958	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6959	if (fSupportsNextRipSave)
6960	{
6961	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
6962	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6963	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6964	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
6965	}
6966	else
6967	{
6968	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6969	rcStrict = IEMExecOne(pVCpu);
6970	}
6971
6972	if (rcStrict == VINF_SUCCESS)
6973	pSvmTransient->fUpdateTscOffsetting = true;
6974	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6975	{
6976	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6977	rcStrict = VINF_SUCCESS;
6978	}
6979	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6980	return rcStrict;
6981	}
6982
6983
6984	/**
6985	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6986	*/
6987	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6988	{
6989	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6990
6991	VBOXSTRICTRC rcStrict;
6992	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6993	if (fSupportsNextRipSave)
6994	{
6995	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6996	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6997	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6998	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
6999	}
7000	else
7001	{
7002	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7003	rcStrict = IEMExecOne(pVCpu);
7004	}
7005
7006	if (rcStrict == VINF_IEM_RAISED_XCPT)
7007	{
7008	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7009	rcStrict = VINF_SUCCESS;
7010	}
7011	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7012	return rcStrict;
7013	}
7014
7015
7016	/**
7017	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7018	*/
7019	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7020	{
7021	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7022	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7023
7024	VBOXSTRICTRC rcStrict;
7025	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7026	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7027	if ( fSupportsDecodeAssists
7028	&& fSupportsNextRipSave)
7029	{
7030	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7031	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7032	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7033	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7034	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7035	}
7036	else
7037	{
7038	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7039	rcStrict = IEMExecOne(pVCpu);
7040	}
7041
7042	if (rcStrict == VINF_IEM_RAISED_XCPT)
7043	{
7044	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7045	rcStrict = VINF_SUCCESS;
7046	}
7047	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7048	return VBOXSTRICTRC_VAL(rcStrict);
7049	}
7050
7051
7052	/**
7053	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7054	*/
7055	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7056	{
7057	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7058
7059	VBOXSTRICTRC rcStrict;
7060	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7061	if (fSupportsNextRipSave)
7062	{
7063	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7064	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7065	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7066	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7067	}
7068	else
7069	{
7070	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7071	rcStrict = IEMExecOne(pVCpu);
7072	}
7073
7074	if ( rcStrict == VINF_EM_HALT
7075	\|\| rcStrict == VINF_SUCCESS)
7076	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7077	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7078	{
7079	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7080	rcStrict = VINF_SUCCESS;
7081	}
7082	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7083	if (rcStrict != VINF_SUCCESS)
7084	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7085	return VBOXSTRICTRC_VAL(rcStrict);;
7086	}
7087
7088
7089	/**
7090	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7091	*/
7092	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7093	{
7094	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7095
7096	/*
7097	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7098	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
7099	* segment override prefix or other prefixes might be used, in which case we fallback to
7100	* IEMExecOne() to figure out.
7101	*/
7102	VBOXSTRICTRC rcStrict;
7103	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7104	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7105	if (cbInstr)
7106	{
7107	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7108	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7109	}
7110	else
7111	{
7112	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7113	rcStrict = IEMExecOne(pVCpu);
7114	}
7115
7116	if (rcStrict == VINF_IEM_RAISED_XCPT)
7117	{
7118	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7119	rcStrict = VINF_SUCCESS;
7120	}
7121	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7122	return rcStrict;
7123	}
7124
7125
7126	/**
7127	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7128	*/
7129	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7130	{
7131	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7132
7133	VBOXSTRICTRC rcStrict;
7134	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7135	if (fSupportsNextRipSave)
7136	{
7137	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7138	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7139	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7140	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7141	}
7142	else
7143	{
7144	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7145	rcStrict = IEMExecOne(pVCpu);
7146	}
7147
7148	if ( rcStrict == VINF_EM_HALT
7149	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7150	rcStrict = VINF_SUCCESS;
7151	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7152	{
7153	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7154	rcStrict = VINF_SUCCESS;
7155	}
7156	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7157	return rcStrict;
7158	}
7159
7160
7161	/**
7162	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7163	* \#VMEXIT.
7164	*/
7165	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7166	{
7167	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7168	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7169	return VINF_EM_RESET;
7170	}
7171
7172
7173	/**
7174	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7175	*/
7176	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7177	{
7178	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7179	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7180	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7181	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7182	RT_NOREF(pVmcb);
7183	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7184	return VERR_SVM_UNEXPECTED_EXIT;
7185	}
7186
7187
7188	/**
7189	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7190	*/
7191	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7192	{
7193	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7194
7195	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7196	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7197	#ifdef VBOX_WITH_STATISTICS
7198	switch (pSvmTransient->u64ExitCode)
7199	{
7200	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7201	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7202	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7203	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7204	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7205	}
7206	#endif
7207
7208	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7209	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7210	if ( fSupportsDecodeAssists
7211	&& fSupportsNextRipSave)
7212	{
7213	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7214	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7215	if (fMovCRx)
7216	{
7217	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
7218	\| CPUMCTX_EXTRN_APIC_TPR);
7219	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7220	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7221	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7222	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7223	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7224	return VBOXSTRICTRC_VAL(rcStrict);
7225	}
7226	/* else: SMSW instruction, fall back below to IEM for this. */
7227	}
7228
7229	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7230	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7231	AssertMsg( rcStrict == VINF_SUCCESS
7232	\|\| rcStrict == VINF_PGM_SYNC_CR3
7233	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
7234	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7235	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7236	if (rcStrict == VINF_IEM_RAISED_XCPT)
7237	{
7238	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7239	rcStrict = VINF_SUCCESS;
7240	}
7241	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7242	return rcStrict;
7243	}
7244
7245
7246	/**
7247	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7248	*/
7249	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7250	{
7251	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7252
7253	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7254	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7255	Assert(iCrReg <= 15);
7256
7257	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7258	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7259	bool fDecodedInstr = false;
7260	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7261	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7262	if ( fSupportsDecodeAssists
7263	&& fSupportsNextRipSave)
7264	{
7265	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7266	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7267	if (fMovCRx)
7268	{
7269	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
7270	\| CPUMCTX_EXTRN_APIC_TPR);
7271	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7272	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7273	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7274	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7275	fDecodedInstr = true;
7276	}
7277	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7278	}
7279
7280	if (!fDecodedInstr)
7281	{
7282	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7283	Log4Func(("iCrReg=%#x\n", iCrReg));
7284	rcStrict = IEMExecOne(pVCpu);
7285	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7286	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7287	rcStrict = VERR_EM_INTERPRETER;
7288	}
7289
7290	if (rcStrict == VINF_SUCCESS)
7291	{
7292	switch (iCrReg)
7293	{
7294	case 0:
7295	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7296	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7297	break;
7298
7299	case 2:
7300	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7302	break;
7303
7304	case 3:
7305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7307	break;
7308
7309	case 4:
7310	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7312	break;
7313
7314	case 8:
7315	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7317	break;
7318
7319	default:
7320	{
7321	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7322	pSvmTransient->u64ExitCode, iCrReg));
7323	break;
7324	}
7325	}
7326	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7327	}
7328	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7329	{
7330	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7331	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7332	rcStrict = VINF_SUCCESS;
7333	}
7334	else
7335	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
7336	return rcStrict;
7337	}
7338
7339
7340	/**
7341	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7342	*
7343	* @returns Strict VBox status code.
7344	* @param pVCpu The cross context virtual CPU structure.
7345	* @param pVmcb Pointer to the VM control block.
7346	*/
7347	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7348	{
7349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7350	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7351
7352	VBOXSTRICTRC rcStrict;
7353	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7354	if (fSupportsNextRipSave)
7355	{
7356	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7357	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7358	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7359	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7360	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7361	}
7362	else
7363	{
7364	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7365	rcStrict = IEMExecOne(pVCpu);
7366	}
7367
7368	AssertMsg( rcStrict == VINF_SUCCESS
7369	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7370	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
7371	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7372
7373	if (rcStrict == VINF_IEM_RAISED_XCPT)
7374	{
7375	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7376	rcStrict = VINF_SUCCESS;
7377	}
7378	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7379	return rcStrict;
7380	}
7381
7382
7383	/**
7384	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7385	*
7386	* @returns Strict VBox status code.
7387	* @param pVCpu The cross context virtual CPU structure.
7388	* @param pVmcb Pointer to the VM control block.
7389	* @param pSvmTransient Pointer to the SVM-transient structure.
7390	*/
7391	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7392	{
7393	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7394	uint32_t const idMsr = pCtx->ecx;
7395	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7396	Log4Func(("idMsr=%#RX32\n", idMsr));
7397
7398	/*
7399	* Handle TPR patching MSR writes.
7400	* We utilitize the LSTAR MSR for patching.
7401	*/
7402	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7403	if ( idMsr == MSR_K8_LSTAR
7404	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7405	{
7406	unsigned cbInstr;
7407	if (fSupportsNextRipSave)
7408	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7409	else
7410	{
7411	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
7412	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
7413	if ( rc == VINF_SUCCESS
7414	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
7415	Assert(cbInstr > 0);
7416	else
7417	cbInstr = 0;
7418	}
7419
7420	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7421	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7422	{
7423	int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
7424	AssertRCReturn(rc, rc);
7425	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7426	}
7427
7428	int rc = VINF_SUCCESS;
7429	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7430	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7431	return rc;
7432	}
7433
7434	/*
7435	* Handle regular MSR writes.
7436	*/
7437	VBOXSTRICTRC rcStrict;
7438	if (fSupportsNextRipSave)
7439	{
7440	/** @todo Optimize this: We don't need to get much of the MSR state here
7441	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7442	* clear the applicable extern flags. */
7443	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7444	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7445	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7446	}
7447	else
7448	{
7449	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7450	rcStrict = IEMExecOne(pVCpu);
7451	}
7452
7453	AssertMsg( rcStrict == VINF_SUCCESS
7454	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7455	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
7456	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7457
7458	if (rcStrict == VINF_SUCCESS)
7459	{
7460	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7461	if ( idMsr >= MSR_IA32_X2APIC_START
7462	&& idMsr <= MSR_IA32_X2APIC_END)
7463	{
7464	/*
7465	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7466	* When full APIC register virtualization is implemented we'll have to make sure
7467	* APIC state is saved from the VMCB before IEM changes it.
7468	*/
7469	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7470	}
7471	else
7472	{
7473	switch (idMsr)
7474	{
7475	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7476	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7477	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7478	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7479	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7480	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7481	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7482	}
7483	}
7484	}
7485	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7486	{
7487	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7488	rcStrict = VINF_SUCCESS;
7489	}
7490	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7491	return rcStrict;
7492	}
7493
7494
7495	/**
7496	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7497	* \#VMEXIT.
7498	*/
7499	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7500	{
7501	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7502
7503	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7504	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7505	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7506
7507	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7508	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7509	}
7510
7511
7512	/**
7513	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7514	*/
7515	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7516	{
7517	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7518	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7519
7520	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7521
7522	/** @todo Stepping with nested-guest. */
7523	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7524	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7525	{
7526	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7527	if (pSvmTransient->fWasGuestDebugStateActive)
7528	{
7529	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7530	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7531	return VERR_SVM_UNEXPECTED_EXIT;
7532	}
7533
7534	/*
7535	* Lazy DR0-3 loading.
7536	*/
7537	if (!pSvmTransient->fWasHyperDebugStateActive)
7538	{
7539	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7540	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7541
7542	/* Don't intercept DRx read and writes. */
7543	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7544	pVmcb->ctrl.u16InterceptRdDRx = 0;
7545	pVmcb->ctrl.u16InterceptWrDRx = 0;
7546	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7547
7548	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7549	VMMRZCallRing3Disable(pVCpu);
7550	HM_DISABLE_PREEMPT(pVCpu);
7551
7552	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7553	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7554	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7555
7556	HM_RESTORE_PREEMPT();
7557	VMMRZCallRing3Enable(pVCpu);
7558
7559	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7560	return VINF_SUCCESS;
7561	}
7562	}
7563
7564	/*
7565	* Interpret the read/writing of DRx.
7566	*/
7567	/** @todo Decode assist. */
7568	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
7569	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7570	if (RT_LIKELY(rc == VINF_SUCCESS))
7571	{
7572	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7573	/** @todo CPUM should set this flag! */
7574	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7575	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7576	}
7577	else
7578	Assert(rc == VERR_EM_INTERPRETER);
7579	return rc;
7580	}
7581
7582
7583	/**
7584	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7585	*/
7586	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7587	{
7588	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7589	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7590	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7592	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7593	return rc;
7594	}
7595
7596
7597	/**
7598	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7599	*/
7600	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7601	{
7602	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7603	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7604
7605	/** @todo decode assists... */
7606	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7607	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7608	{
7609	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7610	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7611	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7612	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7613	{
7614	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7615	hmR0SvmUpdateVmRunFunction(pVCpu);
7616	}
7617	}
7618	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7619	{
7620	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7621	rcStrict = VINF_SUCCESS;
7622	}
7623	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7624	return rcStrict;
7625	}
7626
7627
7628	/**
7629	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7630	*/
7631	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7632	{
7633	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7634	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
7635
7636	/* I/O operation lookup arrays. */
7637	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7638	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7639	the result (in AL/AX/EAX). */
7640	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7641	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7642	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7643
7644	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7645
7646	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7647	SVMIOIOEXITINFO IoExitInfo;
7648	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7649	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7650	uint32_t cbValue = s_aIOSize[uIOWidth];
7651	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7652
7653	if (RT_UNLIKELY(!cbValue))
7654	{
7655	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7656	return VERR_EM_INTERPRETER;
7657	}
7658
7659	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
7660	VBOXSTRICTRC rcStrict;
7661	PCEMEXITREC pExitRec = NULL;
7662	if ( !pVCpu->hm.s.fSingleInstruction
7663	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7664	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7665	!IoExitInfo.n.u1Str
7666	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7667	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7668	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7669	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7670	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7671	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7672	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7673	if (!pExitRec)
7674	{
7675	bool fUpdateRipAlready = false;
7676	if (IoExitInfo.n.u1Str)
7677	{
7678	/* INS/OUTS - I/O String instruction. */
7679	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7680	* in EXITINFO1? Investigate once this thing is up and running. */
7681	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7682	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7683	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7684	static IEMMODE const s_aenmAddrMode[8] =
7685	{
7686	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7687	};
7688	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7689	if (enmAddrMode != (IEMMODE)-1)
7690	{
7691	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7692	if (cbInstr <= 15 && cbInstr >= 1)
7693	{
7694	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7695	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7696	{
7697	/* Don't know exactly how to detect whether u3Seg is valid, currently
7698	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7699	2384 Opterons when only checking NRIP. */
7700	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7701	if ( fSupportsNextRipSave
7702	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7703	{
7704	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7705	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7706	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7707	IoExitInfo.n.u3Seg, true /fIoChecked/);
7708	}
7709	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7710	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7711	X86_SREG_DS, true /fIoChecked/);
7712	else
7713	rcStrict = IEMExecOne(pVCpu);
7714	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7715	}
7716	else
7717	{
7718	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7719	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7720	true /fIoChecked/);
7721	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7722	}
7723	}
7724	else
7725	{
7726	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7727	rcStrict = IEMExecOne(pVCpu);
7728	}
7729	}
7730	else
7731	{
7732	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7733	rcStrict = IEMExecOne(pVCpu);
7734	}
7735	fUpdateRipAlready = true;
7736	if (rcStrict == VINF_IEM_RAISED_XCPT)
7737	{
7738	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7739	rcStrict = VINF_SUCCESS;
7740	}
7741	}
7742	else
7743	{
7744	/* IN/OUT - I/O instruction. */
7745	Assert(!IoExitInfo.n.u1Rep);
7746
7747	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7748	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7749	{
7750	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7751	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7752	&& !pCtx->eflags.Bits.u1TF)
7753	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7754	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7755	}
7756	else
7757	{
7758	uint32_t u32Val = 0;
7759	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7760	if (IOM_SUCCESS(rcStrict))
7761	{
7762	/* Save result of I/O IN instr. in AL/AX/EAX. */
7763	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7764	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7765	}
7766	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7767	&& !pCtx->eflags.Bits.u1TF)
7768	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7769
7770	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7771	}
7772	}
7773
7774	if (IOM_SUCCESS(rcStrict))
7775	{
7776	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7777	if (!fUpdateRipAlready)
7778	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7779
7780	/*
7781	* If any I/O breakpoints are armed, we need to check if one triggered
7782	* and take appropriate action.
7783	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7784	*/
7785	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7786	* execution engines about whether hyper BPs and such are pending. */
7787	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7788	uint32_t const uDr7 = pCtx->dr[7];
7789	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7790	&& X86_DR7_ANY_RW_IO(uDr7)
7791	&& (pCtx->cr4 & X86_CR4_DE))
7792	\|\| DBGFBpIsHwIoArmed(pVM)))
7793	{
7794	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7795	VMMRZCallRing3Disable(pVCpu);
7796	HM_DISABLE_PREEMPT(pVCpu);
7797
7798	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7799	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7800
7801	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7802	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7803	{
7804	/* Raise #DB. */
7805	pVmcb->guest.u64DR6 = pCtx->dr[6];
7806	pVmcb->guest.u64DR7 = pCtx->dr[7];
7807	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7808	hmR0SvmSetPendingXcptDB(pVCpu);
7809	}
7810	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7811	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7812	else if ( rcStrict2 != VINF_SUCCESS
7813	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7814	rcStrict = rcStrict2;
7815	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7816
7817	HM_RESTORE_PREEMPT();
7818	VMMRZCallRing3Enable(pVCpu);
7819	}
7820
7821	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7822	}
7823	#ifdef VBOX_STRICT
7824	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7825	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7826	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7827	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7828	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7829	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7830	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7831	else
7832	{
7833	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7834	* statuses, that the VMM device and some others may return. See
7835	* IOM_SUCCESS() for guidance. */
7836	AssertMsg( RT_FAILURE(rcStrict)
7837	\|\| rcStrict == VINF_SUCCESS
7838	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7839	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7840	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7841	\|\| rcStrict == VINF_EM_DBG_STEPPED
7842	\|\| rcStrict == VINF_EM_RAW_TO_R3
7843	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7844	}
7845	#endif
7846	}
7847	else
7848	{
7849	/*
7850	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7851	*/
7852	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7853	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7854	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7855	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7856	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7857	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7858	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7859
7860	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7861	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7862
7863	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7864	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7865	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7866	}
7867	return rcStrict;
7868	}
7869
7870
7871	/**
7872	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7873	*/
7874	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7875	{
7876	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7877	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7878	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7879
7880	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7881	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7882	Assert(pVM->hmr0.s.fNestedPaging);
7883
7884	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7885	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7886	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7887	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7888	thus intentionally not copied into u32ErrCode. */
7889
7890	Log4Func(("#NPF at CS:RIP=%04x:%RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7891	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7892
7893	/*
7894	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7895	*/
7896	if ( pVM->hm.s.fTprPatchingAllowed
7897	&& (GCPhysFaultAddr & GUEST_PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7898	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7899	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7900	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7901	&& !CPUMIsGuestInLongModeEx(pCtx)
7902	&& !CPUMGetGuestCPL(pVCpu)
7903	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7904	{
7905	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7906	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
7907
7908	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7909	{
7910	/* Only attempt to patch the instruction once. */
7911	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7912	if (!pPatch)
7913	return VINF_EM_HM_PATCH_TPR_INSTR;
7914	}
7915	}
7916
7917	/*
7918	* Determine the nested paging mode.
7919	*/
7920	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7921	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7922
7923	/*
7924	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7925	*/
7926	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7927	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7928	{
7929	/*
7930	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7931	* injecting the original pending event would most likely cause the same MMIO #NPF.
7932	*/
7933	if (pVCpu->hm.s.Event.fPending)
7934	{
7935	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7936	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7937	}
7938
7939	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7940	VBOXSTRICTRC rcStrict;
7941	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7942	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7943	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7944	if (!pExitRec)
7945	{
7946
7947	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, pCtx, GCPhysFaultAddr, u32ErrCode);
7948
7949	/*
7950	* If we succeed, resume guest execution.
7951	*
7952	* If we fail in interpreting the instruction because we couldn't get the guest
7953	* physical address of the page containing the instruction via the guest's page
7954	* tables (we would invalidate the guest page in the host TLB), resume execution
7955	* which would cause a guest page fault to let the guest handle this weird case.
7956	*
7957	* See @bugref{6043}.
7958	*/
7959	if ( rcStrict == VINF_SUCCESS
7960	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7961	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
7962	{
7963	/* Successfully handled MMIO operation. */
7964	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7965	rcStrict = VINF_SUCCESS;
7966	}
7967	}
7968	else
7969	{
7970	/*
7971	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7972	*/
7973	Assert(pCtx == &pVCpu->cpum.GstCtx);
7974	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7975	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7976	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
7977
7978	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7979	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7980
7981	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7982	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7983	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7984	}
7985	return rcStrict;
7986	}
7987
7988	/*
7989	* Nested page-fault.
7990	*/
7991	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
7992	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, pCtx, GCPhysFaultAddr);
7993	TRPMResetTrap(pVCpu);
7994
7995	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
7996
7997	/*
7998	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
7999	*/
8000	if ( rc == VINF_SUCCESS
8001	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8002	\|\| rc == VERR_PAGE_NOT_PRESENT)
8003	{
8004	/* We've successfully synced our shadow page tables. */
8005	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8006	rc = VINF_SUCCESS;
8007	}
8008
8009	/*
8010	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
8011	* re-inject the original event.
8012	*/
8013	if (pVCpu->hm.s.Event.fPending)
8014	{
8015	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8016
8017	/*
8018	* If the #NPF handler requested emulation of the instruction, ignore it.
8019	* We need to re-inject the original event so as to not lose it.
8020	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8021	* LiveCD is broken for other reasons).
8022	*/
8023	if (rc == VINF_EM_RAW_EMULATE_INSTR)
8024	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8025	}
8026
8027	return rc;
8028	}
8029
8030
8031	/**
8032	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8033	* \#VMEXIT.
8034	*/
8035	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8036	{
8037	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8038	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8039
8040	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8041	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8042	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8043
8044	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8045	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8046	return VINF_SUCCESS;
8047	}
8048
8049
8050	/**
8051	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8052	* \#VMEXIT.
8053	*/
8054	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8055	{
8056	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8057	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8058
8059	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8060	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8061	#endif
8062
8063	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
8064	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8065	{
8066	/*
8067	* AMD-V provides us with the exception which caused the TS; we collect
8068	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8069	*/
8070	Log4Func(("TS occurred during event delivery\n"));
8071	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8072	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8073	}
8074
8075	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8076	* emulation. */
8077	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8078	return VERR_EM_INTERPRETER;
8079	}
8080
8081
8082	/**
8083	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8084	*/
8085	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8086	{
8087	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8088	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8089
8090	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8091	if (pVM->hm.s.fTprPatchingAllowed)
8092	{
8093	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8094	if (rc != VERR_NOT_FOUND)
8095	{
8096	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8097	return rc;
8098	}
8099	}
8100
8101	if (EMAreHypercallInstructionsEnabled(pVCpu))
8102	{
8103	unsigned cbInstr;
8104	if (hmR0SvmSupportsNextRipSave(pVCpu))
8105	{
8106	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8107	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8108	}
8109	else
8110	{
8111	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8112	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8113	if ( rc == VINF_SUCCESS
8114	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
8115	Assert(cbInstr > 0);
8116	else
8117	cbInstr = 0;
8118	}
8119
8120	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8121	if (RT_SUCCESS(rcStrict))
8122	{
8123	/* Only update the RIP if we're continuing guest execution and not in the case
8124	of say VINF_GIM_R3_HYPERCALL. */
8125	if (rcStrict == VINF_SUCCESS)
8126	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8127
8128	return VBOXSTRICTRC_VAL(rcStrict);
8129	}
8130	else
8131	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8132	}
8133
8134	hmR0SvmSetPendingXcptUD(pVCpu);
8135	return VINF_SUCCESS;
8136	}
8137
8138
8139	/**
8140	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8141	*/
8142	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8143	{
8144	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8145
8146	unsigned cbInstr;
8147	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8148	if (fSupportsNextRipSave)
8149	{
8150	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8151	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8152	}
8153	else
8154	{
8155	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8156	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8157	if ( rc == VINF_SUCCESS
8158	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
8159	Assert(cbInstr > 0);
8160	else
8161	cbInstr = 0;
8162	}
8163
8164	/** @todo The guest has likely hit a contended spinlock. We might want to
8165	* poke a schedule different guest VCPU. */
8166	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8167	return VINF_EM_RAW_INTERRUPT;
8168	}
8169
8170
8171	/**
8172	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8173	* \#VMEXIT.
8174	*/
8175	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8176	{
8177	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8178	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8179	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8180
8181	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8182	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8183	}
8184
8185
8186	/**
8187	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8188	*/
8189	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8190	{
8191	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8192
8193	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8194	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8195	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8196
8197	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8198	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8199	* handler executes. Yes, x86 is ugly.
8200	*/
8201	return VINF_EM_RAW_EMULATE_INSTR;
8202	}
8203
8204
8205	/**
8206	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8207	* Conditional \#VMEXIT.
8208	*/
8209	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8210	{
8211	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8212	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8213	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8214
8215	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8216	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8217	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8218	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8219	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8220	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8221
8222	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
8223	if (pVM->hmr0.s.fNestedPaging)
8224	{
8225	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8226	if ( !pSvmTransient->fVectoringDoublePF
8227	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8228	{
8229	/* A genuine guest #PF, reflect it to the guest. */
8230	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8231	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8232	uFaultAddress, uErrCode));
8233	}
8234	else
8235	{
8236	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8237	hmR0SvmSetPendingXcptDF(pVCpu);
8238	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8239	}
8240	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8241	return VINF_SUCCESS;
8242	}
8243	#endif
8244
8245	Assert(!pVM->hmr0.s.fNestedPaging);
8246
8247	/*
8248	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8249	*/
8250	if ( pVM->hm.s.fTprPatchingAllowed
8251	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8252	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8253	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8254	&& !CPUMIsGuestInLongModeEx(pCtx)
8255	&& !CPUMGetGuestCPL(pVCpu)
8256	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8257	{
8258	RTGCPHYS GCPhysApicBase;
8259	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
8260	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
8261
8262	/* Check if the page at the fault-address is the APIC base. */
8263	PGMPTWALK Walk;
8264	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8265	if ( rc2 == VINF_SUCCESS
8266	&& Walk.GCPhys == GCPhysApicBase)
8267	{
8268	/* Only attempt to patch the instruction once. */
8269	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8270	if (!pPatch)
8271	return VINF_EM_HM_PATCH_TPR_INSTR;
8272	}
8273	}
8274
8275	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8276	pCtx->rip, uErrCode, pCtx->cr3));
8277
8278	/*
8279	* If it's a vectoring #PF, emulate injecting the original event injection as
8280	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8281	* event injection that caused a #PF. See @bugref{6607}.
8282	*/
8283	if (pSvmTransient->fVectoringPF)
8284	{
8285	Assert(pVCpu->hm.s.Event.fPending);
8286	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8287	}
8288
8289	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8290	int rc = PGMTrap0eHandler(pVCpu, uErrCode, pCtx, (RTGCPTR)uFaultAddress);
8291
8292	Log4Func(("#PF: rc=%Rrc\n", rc));
8293
8294	if (rc == VINF_SUCCESS)
8295	{
8296	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8297	TRPMResetTrap(pVCpu);
8298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8300	return rc;
8301	}
8302
8303	if (rc == VINF_EM_RAW_GUEST_TRAP)
8304	{
8305	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8306
8307	/*
8308	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8309	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8310	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8311	*/
8312	if ( !pSvmTransient->fVectoringDoublePF
8313	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8314	{
8315	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
8316	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8317	TRPMResetTrap(pVCpu);
8318
8319	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8320	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8321	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8322	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8323	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8324	#endif
8325
8326	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8327	}
8328	else
8329	{
8330	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8331	TRPMResetTrap(pVCpu);
8332	hmR0SvmSetPendingXcptDF(pVCpu);
8333	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8334	}
8335
8336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8337	return VINF_SUCCESS;
8338	}
8339
8340	TRPMResetTrap(pVCpu);
8341	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8342	return rc;
8343	}
8344
8345
8346
8347	/**
8348	* \#VMEXIT handler for division overflow exceptions (SVM_EXIT_XCPT_1).
8349	* Conditional \#VMEXIT.
8350	*/
8351	HMSVM_EXIT_DECL hmR0SvmExitXcptDE(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8352	{
8353	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8354	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8356
8357	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8358	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8359	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8360
8361	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8362	if (pVCpu->hm.s.fGCMTrapXcptDE)
8363	{
8364	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8365	uint8_t cbInstr = 0;
8366	VBOXSTRICTRC rcStrict = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8367	if (rcStrict == VINF_SUCCESS)
8368	rc = VINF_SUCCESS; /* Restart instruction with modified guest register context. */
8369	else if (rcStrict == VERR_NOT_FOUND)
8370	rc = VERR_NOT_FOUND; /* Deliver the exception. */
8371	else
8372	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8373	}
8374
8375	/* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
8376	if (RT_FAILURE(rc))
8377	{
8378	hmR0SvmSetPendingXcptDE(pVCpu);
8379	rc = VINF_SUCCESS;
8380	}
8381
8382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8383	return rc;
8384	}
8385
8386
8387	/**
8388	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8389	* Conditional \#VMEXIT.
8390	*/
8391	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8392	{
8393	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8394	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8395	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8396
8397	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8398	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8399	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8400
8401	/** @todo if we accumulate more optional stuff here, we ought to combine the
8402	* reading of opcode bytes to avoid doing more than once. */
8403
8404	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8405	if (pVCpu->hm.s.fGIMTrapXcptUD)
8406	{
8407	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8408	uint8_t cbInstr = 0;
8409	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8410	if (rcStrict == VINF_SUCCESS)
8411	{
8412	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8413	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8414	rcStrict = VINF_SUCCESS;
8415	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8416	}
8417	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8418	rcStrict = VINF_SUCCESS;
8419	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8420	rcStrict = VINF_GIM_R3_HYPERCALL;
8421	else
8422	{
8423	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8424	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8425	}
8426	}
8427
8428	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8429	{
8430	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
8431	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8432	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8433	{
8434	/* Ideally, IEM should just handle all these special #UD situations, but
8435	we don't quite trust things to behave optimially when doing that. So,
8436	for now we'll restrict ourselves to a handful of possible sysenter and
8437	sysexit encodings that we filter right here. */
8438	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8439	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8440	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8441	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8442	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8443	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8444	{
8445	cbInstr = cbMin;
8446	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8447	AssertRCStmt(rc2, cbInstr = 0);
8448	}
8449	else
8450	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8451	if ( cbInstr == 0 /* read error */
8452	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8453	\|\| ( uCpl == 0
8454	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8455	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8456	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
8457	{
8458	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8459	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8460	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8461	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, GCPtrInstr, abInstr, cbInstr);
8462	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8463	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8464	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8465	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8466	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8467	if (rcStrict == VINF_IEM_RAISED_XCPT)
8468	rcStrict = VINF_SUCCESS;
8469	return rcStrict;
8470	}
8471	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8472	}
8473	else
8474	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8475	}
8476
8477	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8478	if (RT_FAILURE(rcStrict))
8479	{
8480	hmR0SvmSetPendingXcptUD(pVCpu);
8481	rcStrict = VINF_SUCCESS;
8482	}
8483
8484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8485	return rcStrict;
8486	}
8487
8488
8489	/**
8490	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8491	* Conditional \#VMEXIT.
8492	*/
8493	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8494	{
8495	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8496	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8497	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8498
8499	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8500	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8501
8502	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8503	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8504
8505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8506
8507	if (!(pCtx->cr0 & X86_CR0_NE))
8508	{
8509	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8510	unsigned cbInstr;
8511	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8512	if (RT_SUCCESS(rc))
8513	{
8514	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8515	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8516	if (RT_SUCCESS(rc))
8517	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8518	}
8519	else
8520	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8521	return rc;
8522	}
8523
8524	hmR0SvmSetPendingXcptMF(pVCpu);
8525	return VINF_SUCCESS;
8526	}
8527
8528
8529	/**
8530	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8531	* \#VMEXIT.
8532	*/
8533	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8534	{
8535	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8536	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8537	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8539
8540	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8541	{
8542	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8543	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8544	}
8545
8546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8547
8548	/*
8549	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8550	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8551	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8552	*/
8553	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8554	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8555	int rc = DBGFTrap01Handler(pVM, pVCpu, &pVCpu->cpum.GstCtx, pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8556	if (rc == VINF_EM_RAW_GUEST_TRAP)
8557	{
8558	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8559	if (CPUMIsHyperDebugStateActive(pVCpu))
8560	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
8561
8562	/* Reflect the exception back to the guest. */
8563	hmR0SvmSetPendingXcptDB(pVCpu);
8564	rc = VINF_SUCCESS;
8565	}
8566
8567	/*
8568	* Update DR6.
8569	*/
8570	if (CPUMIsHyperDebugStateActive(pVCpu))
8571	{
8572	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8573	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8574	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8575	}
8576	else
8577	{
8578	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8579	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8580	}
8581
8582	return rc;
8583	}
8584
8585
8586	/**
8587	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8588	* Conditional \#VMEXIT.
8589	*/
8590	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8591	{
8592	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8593	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8594	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8595
8596	SVMEVENT Event;
8597	Event.u = 0;
8598	Event.n.u1Valid = 1;
8599	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8600	Event.n.u8Vector = X86_XCPT_AC;
8601	Event.n.u1ErrorCodeValid = 1;
8602	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8603	return VINF_SUCCESS;
8604	}
8605
8606
8607	/**
8608	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8609	* Conditional \#VMEXIT.
8610	*/
8611	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8612	{
8613	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8614	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8615	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8617
8618	VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx);
8619	if (rc == VINF_EM_RAW_GUEST_TRAP)
8620	{
8621	SVMEVENT Event;
8622	Event.u = 0;
8623	Event.n.u1Valid = 1;
8624	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8625	Event.n.u8Vector = X86_XCPT_BP;
8626	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8627	rc = VINF_SUCCESS;
8628	}
8629
8630	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
8631	return rc;
8632	}
8633
8634
8635	/**
8636	* Hacks its way around the lovely mesa driver's backdoor accesses.
8637	*
8638	* @sa hmR0VmxHandleMesaDrvGp
8639	*/
8640	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8641	{
8642	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
8643	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8644	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8645	RT_NOREF(pCtx, pVmcb);
8646
8647	/* For now we'll just skip the instruction. */
8648	hmR0SvmAdvanceRip(pVCpu, 1);
8649	return VINF_SUCCESS;
8650	}
8651
8652
8653	/**
8654	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8655	* backdoor logging w/o checking what it is running inside.
8656	*
8657	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8658	* backdoor port and magic numbers loaded in registers.
8659	*
8660	* @returns true if it is, false if it isn't.
8661	* @sa hmR0VmxIsMesaDrvGp
8662	*/
8663	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8664	{
8665	/* Check magic and port. */
8666	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
8667	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
8668	if (pCtx->dx != UINT32_C(0x5658))
8669	return false;
8670	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8671	return false;
8672
8673	/* Check that it is #GP(0). */
8674	if (pVmcb->ctrl.u64ExitInfo1 != 0)
8675	return false;
8676
8677	/* Flat ring-3 CS. */
8678	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
8679	if (pVmcb->guest.u8CPL != 3)
8680	return false;
8681	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8682	return false;
8683
8684	/* 0xed: IN eAX,dx */
8685	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8686	{
8687	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
8688	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8689	uint8_t abInstr[1];
8690	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8691	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
8692	if (RT_FAILURE(rc))
8693	return false;
8694	if (abInstr[0] != 0xed)
8695	return false;
8696	}
8697	else
8698	{
8699	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
8700	if (pVmcb->ctrl.abInstr[0] != 0xed)
8701	return false;
8702	}
8703	return true;
8704	}
8705
8706
8707	/**
8708	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8709	* Conditional \#VMEXIT.
8710	*/
8711	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8712	{
8713	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8714	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8716
8717	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8718	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8719
8720	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8721	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8722	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8723	{
8724	SVMEVENT Event;
8725	Event.u = 0;
8726	Event.n.u1Valid = 1;
8727	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8728	Event.n.u8Vector = X86_XCPT_GP;
8729	Event.n.u1ErrorCodeValid = 1;
8730	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8731	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8732	return VINF_SUCCESS;
8733	}
8734	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8735	}
8736
8737
8738	/**
8739	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8740	*/
8741	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8742	{
8743	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8744	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8745
8746	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8747	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8748	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8749	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8750	Assert(uVector <= X86_XCPT_LAST);
8751	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8752
8753	SVMEVENT Event;
8754	Event.u = 0;
8755	Event.n.u1Valid = 1;
8756	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8757	Event.n.u8Vector = uVector;
8758	switch (uVector)
8759	{
8760	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8761	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8762	case X86_XCPT_DF:
8763	case X86_XCPT_TS:
8764	case X86_XCPT_NP:
8765	case X86_XCPT_SS:
8766	case X86_XCPT_GP:
8767	case X86_XCPT_AC:
8768	{
8769	Event.n.u1ErrorCodeValid = 1;
8770	Event.n.u32ErrorCode = uErrCode;
8771	break;
8772	}
8773	}
8774
8775	#ifdef VBOX_WITH_STATISTICS
8776	switch (uVector)
8777	{
8778	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8779	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8780	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8781	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8782	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8783	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8784	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8785	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8786	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8787	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8788	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8789	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8790	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8791	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8792	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8793	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8794	default:
8795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8796	break;
8797	}
8798	#endif
8799
8800	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8801	return VINF_SUCCESS;
8802	}
8803
8804
8805	/**
8806	* \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8807	*/
8808	HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8809	{
8810	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8811	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8812
8813	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8814	SVMEVENT Event;
8815	Event.u = 0;
8816	Event.n.u1Valid = 1;
8817	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8818	Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8819	Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8820	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8821	return VINF_SUCCESS;
8822	}
8823
8824
8825	/**
8826	* Generic exit handler that interprets the current instruction
8827	*
8828	* Useful exit that only gets triggered by dtrace and the debugger. Caller does
8829	* the exit logging, and this function does the rest.
8830	*/
8831	static VBOXSTRICTRC hmR0SvmExitInterpretInstruction(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient,
8832	uint64_t fExtraImport, uint64_t fHmChanged)
8833	{
8834	#if 1
8835	RT_NOREF(pSvmTransient);
8836	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fExtraImport);
8837	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
8838	if (rcStrict == VINF_SUCCESS)
8839	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RIP);
8840	else
8841	{
8842	Log4Func(("IEMExecOne -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
8843	if (rcStrict == VINF_IEM_RAISED_XCPT)
8844	{
8845	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK \| fHmChanged);
8846	rcStrict = VINF_SUCCESS;
8847	}
8848	else
8849	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged);
8850	}
8851	return rcStrict;
8852	#else
8853	RT_NOREF(pVCpu, pSvmTransient, fExtraImport, fHmChanged);
8854	return VINF_EM_RAW_EMULATE_INSTR;
8855	#endif
8856	}
8857
8858
8859	/**
8860	* \#VMEXIT handler for STR. Conditional \#VMEXIT (debug).
8861	*/
8862	HMSVM_EXIT_DECL hmR0SvmExitTrRead(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8863	{
8864	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8865	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8866	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR, 0);
8867	}
8868
8869
8870	/**
8871	* \#VMEXIT handler for LTR. Conditional \#VMEXIT (OS/2 TLB workaround, debug).
8872	*/
8873	HMSVM_EXIT_DECL hmR0SvmExitTrWrite(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8874	{
8875	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8876
8877	/* Workaround for lack of TLB flushing in OS/2 when returning to protected
8878	mode after a real mode call (like a BIOS call). See ticketref:20625
8879	comment 14. */
8880	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8881	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
8882	{
8883	Log4Func(("%04x:%08RX64 TLB flush\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8884	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
8885	}
8886	else
8887	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8888
8889	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR, HM_CHANGED_GUEST_TR);
8890	}
8891
8892
8893	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8894	/**
8895	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8896	*/
8897	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8898	{
8899	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8900
8901	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8902	Assert(pVmcb);
8903	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8904
8905	VBOXSTRICTRC rcStrict;
8906	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8907	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8908	if (fSupportsNextRipSave)
8909	{
8910	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8911	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8912	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8913	}
8914	else
8915	{
8916	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8917	rcStrict = IEMExecOne(pVCpu);
8918	}
8919
8920	if (rcStrict == VINF_SUCCESS)
8921	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8922	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8923	{
8924	rcStrict = VINF_SUCCESS;
8925	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8926	}
8927	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8928	return rcStrict;
8929	}
8930
8931
8932	/**
8933	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8934	*/
8935	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8936	{
8937	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8938
8939	/*
8940	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8941	* we only intercept STGI when events are pending for GIF to become 1.
8942	*/
8943	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8944	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8945	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8946
8947	VBOXSTRICTRC rcStrict;
8948	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8949	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8950	if (fSupportsNextRipSave)
8951	{
8952	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8953	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8954	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8955	}
8956	else
8957	{
8958	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8959	rcStrict = IEMExecOne(pVCpu);
8960	}
8961
8962	if (rcStrict == VINF_SUCCESS)
8963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8964	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8965	{
8966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8967	rcStrict = VINF_SUCCESS;
8968	}
8969	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8970	return rcStrict;
8971	}
8972
8973
8974	/**
8975	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
8976	*/
8977	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8978	{
8979	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8980
8981	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8982	Assert(pVmcb);
8983	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
8984
8985	VBOXSTRICTRC rcStrict;
8986	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8987	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
8988	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
8989	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
8990	if (fSupportsNextRipSave)
8991	{
8992	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8993	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8994	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
8995	}
8996	else
8997	{
8998	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8999	rcStrict = IEMExecOne(pVCpu);
9000	}
9001
9002	if (rcStrict == VINF_SUCCESS)
9003	{
9004	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
9005	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
9006	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
9007	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
9008	}
9009	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9010	{
9011	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9012	rcStrict = VINF_SUCCESS;
9013	}
9014	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9015	return rcStrict;
9016	}
9017
9018
9019	/**
9020	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
9021	*/
9022	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9023	{
9024	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9025
9026	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9027	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9028
9029	VBOXSTRICTRC rcStrict;
9030	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9031	if (fSupportsNextRipSave)
9032	{
9033	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9034	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9035	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
9036	}
9037	else
9038	{
9039	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9040	rcStrict = IEMExecOne(pVCpu);
9041	}
9042
9043	if (rcStrict == VINF_IEM_RAISED_XCPT)
9044	{
9045	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9046	rcStrict = VINF_SUCCESS;
9047	}
9048	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9049	return rcStrict;
9050	}
9051
9052
9053	/**
9054	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
9055	*/
9056	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9057	{
9058	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9059
9060	VBOXSTRICTRC rcStrict;
9061	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9062	if (fSupportsNextRipSave)
9063	{
9064	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9065	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9066	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9067	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
9068	}
9069	else
9070	{
9071	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9072	rcStrict = IEMExecOne(pVCpu);
9073	}
9074
9075	if (rcStrict == VINF_IEM_RAISED_XCPT)
9076	{
9077	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9078	rcStrict = VINF_SUCCESS;
9079	}
9080	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9081	return rcStrict;
9082	}
9083
9084
9085	/**
9086	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
9087	*/
9088	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9089	{
9090	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9091	/* We shall import the entire state here, just in case we enter and continue execution of
9092	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
9093	could lose lose part of CPU state. */
9094	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
9095
9096	VBOXSTRICTRC rcStrict;
9097	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9098	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
9099	if (fSupportsNextRipSave)
9100	{
9101	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9102	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9103	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
9104	}
9105	else
9106	{
9107	/* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
9108	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
9109	instruction itself, see @bugref{7243#c126} */
9110	rcStrict = IEMExecOneBypassEx(pVCpu, NULL /* pcbWritten */);
9111	}
9112	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
9113
9114	if (rcStrict == VINF_SUCCESS)
9115	{
9116	rcStrict = VINF_SVM_VMRUN;
9117	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9118	}
9119	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9120	{
9121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9122	rcStrict = VINF_SUCCESS;
9123	}
9124	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9125	return rcStrict;
9126	}
9127
9128
9129	/**
9130	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9131	* Unconditional \#VMEXIT.
9132	*/
9133	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9134	{
9135	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9136	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9137
9138	if (pVCpu->hm.s.Event.fPending)
9139	{
9140	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9141	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9142	}
9143
9144	hmR0SvmSetPendingXcptDB(pVCpu);
9145	return VINF_SUCCESS;
9146	}
9147
9148
9149	/**
9150	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9151	* Conditional \#VMEXIT.
9152	*/
9153	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9154	{
9155	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9156	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9157
9158	SVMEVENT Event;
9159	Event.u = 0;
9160	Event.n.u1Valid = 1;
9161	Event.n.u3Type = SVM_EVENT_EXCEPTION;
9162	Event.n.u8Vector = X86_XCPT_BP;
9163	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9164	return VINF_SUCCESS;
9165	}
9166	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9167
9168	/** @} */
9169

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 100357

Download in other formats: